107 lines
2.6 KiB
TypeScript
107 lines
2.6 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import type { EvalArchiveEntry } from "./archive.js";
|
|
import { computeDriftReportsFromEntries } from "./drift.js";
|
|
|
|
function archiveEntry(
|
|
evaluatedAt: Date,
|
|
overall: number,
|
|
): EvalArchiveEntry {
|
|
return {
|
|
streamId: `${evaluatedAt.toISOString()}:${overall}`,
|
|
evaluatedAt,
|
|
request: {
|
|
requestId: "req-1",
|
|
skill: "onboarding",
|
|
chatroomId: "room-1",
|
|
userId: "user-1",
|
|
nickname: "닉",
|
|
trigger: "user-message",
|
|
},
|
|
response: {
|
|
requestId: "req-1",
|
|
text: "응답",
|
|
confidence: 0.9,
|
|
source: "llm",
|
|
},
|
|
derived: {
|
|
phase: { kind: "photo-pending" },
|
|
intent: { kind: "status-question" },
|
|
shouldRespond: true,
|
|
responseMode: "llm",
|
|
suggestedMaxTokens: 128,
|
|
allowShortResponse: true,
|
|
tier: 0,
|
|
},
|
|
classified: {
|
|
intent: { kind: "status-question" },
|
|
confidence: 0.9,
|
|
source: "regex",
|
|
},
|
|
validation: {
|
|
status: "pass",
|
|
text: "응답",
|
|
reasons: [],
|
|
},
|
|
llmJudge: {
|
|
skipped: false,
|
|
model: "judge",
|
|
endpoint: "local",
|
|
provider: "test",
|
|
scores: {
|
|
friendliness: overall,
|
|
accuracy: overall,
|
|
concision: overall,
|
|
scope: overall,
|
|
overall,
|
|
},
|
|
rationale: "ok",
|
|
raw: "{}",
|
|
},
|
|
latencyMs: 100,
|
|
tier: 0,
|
|
};
|
|
}
|
|
|
|
describe("computeDriftReportsFromEntries", () => {
|
|
it("marks a slice significant when the recent window drops enough", () => {
|
|
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
|
|
const recent = Array.from({ length: 30 }, (_, index) =>
|
|
archiveEntry(new Date(nowMs - index * 60_000), 2),
|
|
);
|
|
const older = Array.from({ length: 40 }, (_, index) =>
|
|
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000 - index * 60_000), 4),
|
|
);
|
|
|
|
const reports = computeDriftReportsFromEntries([...recent, ...older], {
|
|
nowMs,
|
|
thresholdPct: 15,
|
|
});
|
|
|
|
expect(reports).toHaveLength(1);
|
|
expect(reports[0]).toMatchObject({
|
|
recent24hAverage: 2,
|
|
recent24hCount: 30,
|
|
recent7dCount: 70,
|
|
significant: true,
|
|
});
|
|
expect(reports[0]?.dropPct).toBeGreaterThan(15);
|
|
});
|
|
|
|
it("requires enough recent samples for significance", () => {
|
|
const nowMs = Date.parse("2026-05-01T00:00:00.000Z");
|
|
const reports = computeDriftReportsFromEntries(
|
|
[
|
|
archiveEntry(new Date(nowMs), 1),
|
|
archiveEntry(new Date(nowMs - 2 * 24 * 60 * 60 * 1000), 5),
|
|
],
|
|
{
|
|
nowMs,
|
|
thresholdPct: 15,
|
|
},
|
|
);
|
|
|
|
expect(reports[0]?.significant).toBe(false);
|
|
});
|
|
});
|