.
This commit is contained in:
1 parent
1453433b62
commit
da76af3826
65 files changed
+11089
-143
No files matched your search
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { EvalArchiveEntry } from "./archive.js";
|
||||
import { computeDriftReportsFromEntries } from "./drift.js";
|
||||
|
||||
function archiveEntry(
|
||||
evaluatedAt: Date,
|
||||
overall: number,
|
||||
): EvalArchiveEntry {
|
||||
return {
|
||||
streamId: `${evaluatedAt.toISOString()}:${overall}`,
|
||||
evaluatedAt,
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
},
|
||||
response: {
|
||||
requestId: "req-1",
|
||||
text: "응답",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "응답",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: overall,
|
||||
accuracy: overall,
|
||||
concision: overall,
|
||||
scope: overall,
|
||||
overall,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
},
|
||||
latencyMs: 100,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("computeDriftReportsFromEntries", () => {
|
||||
it("marks a slice significant when the recent window drops enough", () => {
|
||||
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
|
||||
const recent = Array.from({ length: 30 }, (_, index) =>
|
||||
archiveEntry(new Date(nowMs - index * 60_000), 2),
|
||||
);
|
||||
const older = Array.from({ length: 40 }, (_, index) =>
|
||||
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000 - index * 60_000), 4),
|
||||
);
|
||||
|
||||
const reports = computeDriftReportsFromEntries([...recent, ...older], {
|
||||
nowMs,
|
||||
thresholdPct: 15,
|
||||
});
|
||||
|
||||
expect(reports).toHaveLength(1);
|
||||
expect(reports[0]).toMatchObject({
|
||||
recent24hAverage: 2,
|
||||
recent24hCount: 30,
|
||||
recent7dCount: 70,
|
||||
significant: true,
|
||||
});
|
||||
expect(reports[0]?.dropPct).toBeGreaterThan(15);
|
||||
});
|
||||
|
||||
it("requires enough recent samples for significance", () => {
|
||||
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
|
||||
const reports = computeDriftReportsFromEntries(
|
||||
[
|
||||
archiveEntry(new Date(nowMs), 1),
|
||||
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000), 5),
|
||||
],
|
||||
{
|
||||
nowMs,
|
||||
thresholdPct: 15,
|
||||
},
|
||||
);
|
||||
|
||||
expect(reports[0]?.significant).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user