This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 1453433b62
commit da76af3826
65 files changed
+11089 -143

No files matched your search

+106
View File
@@ -0,0 +1,106 @@
import { describe, expect, it } from "vitest";
import type { EvalArchiveEntry } from "./archive.js";
import { computeDriftReportsFromEntries } from "./drift.js";
function archiveEntry(
evaluatedAt: Date,
overall: number,
): EvalArchiveEntry {
return {
streamId: `${evaluatedAt.toISOString()}:${overall}`,
evaluatedAt,
request: {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "닉",
trigger: "user-message",
},
response: {
requestId: "req-1",
text: "응답",
confidence: 0.9,
source: "llm",
},
derived: {
phase: { kind: "photo-pending" },
intent: { kind: "status-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
},
classified: {
intent: { kind: "status-question" },
confidence: 0.9,
source: "regex",
},
validation: {
status: "pass",
text: "응답",
reasons: [],
},
llmJudge: {
skipped: false,
model: "judge",
endpoint: "local",
provider: "test",
scores: {
friendliness: overall,
accuracy: overall,
concision: overall,
scope: overall,
overall,
},
rationale: "ok",
raw: "{}",
},
latencyMs: 100,
tier: 0,
};
}
describe("computeDriftReportsFromEntries", () => {
it("marks a slice significant when the recent window drops enough", () => {
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
const recent = Array.from({ length: 30 }, (_, index) =>
archiveEntry(new Date(nowMs - index * 60_000), 2),
);
const older = Array.from({ length: 40 }, (_, index) =>
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000 - index * 60_000), 4),
);
const reports = computeDriftReportsFromEntries([...recent, ...older], {
nowMs,
thresholdPct: 15,
});
expect(reports).toHaveLength(1);
expect(reports[0]).toMatchObject({
recent24hAverage: 2,
recent24hCount: 30,
recent7dCount: 70,
significant: true,
});
expect(reports[0]?.dropPct).toBeGreaterThan(15);
});
it("requires enough recent samples for significance", () => {
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
const reports = computeDriftReportsFromEntries(
[
archiveEntry(new Date(nowMs), 1),
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000), 5),
],
{
nowMs,
thresholdPct: 15,
},
);
expect(reports[0]?.significant).toBe(false);
});
});