Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit a900016be4
60 files changed
+8689 -142

No files matched your search

+139
View File
@@ -0,0 +1,139 @@
import { describe, expect, it } from "vitest";
import type { LlmClient } from "@llm/llm-client";
import type { EvalArchiveEntry } from "../archive.js";
import { runTournament } from "./tournament.js";
function archiveEntry(streamId: string): EvalArchiveEntry {
return {
streamId,
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
request: {
requestId: streamId,
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "닉",
trigger: "user-message",
userMessage: "사진 어디로 보내요?",
},
response: {
requestId: streamId,
text: "이 채팅방에 올려주세요.",
confidence: 0.9,
source: "llm",
},
derived: {
phase: { kind: "photo-pending" },
intent: { kind: "photo-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
},
classified: {
intent: { kind: "photo-question" },
confidence: 0.9,
source: "regex",
},
validation: {
status: "pass",
text: "이 채팅방에 올려주세요.",
reasons: [],
},
llmJudge: {
skipped: false,
model: "judge",
endpoint: "local",
provider: "test",
scores: {
friendliness: 4,
accuracy: 4,
concision: 4,
scope: 4,
overall: 4,
},
rationale: "ok",
raw: "{}",
},
latencyMs: 100,
tier: 0,
};
}
describe("runTournament", () => {
it("runs swapped A/B matches and updates ELO toward winning variants", async () => {
const llmClient = {
hasEndpoints: () => true,
chatCompletion: (
messages: Array<{ content: string; role: string }>,
options?: { jsonMode?: boolean | undefined },
) => {
const content = messages.map((message) => message.content).join("\n");
if (options?.jsonMode) {
const verdict = content.includes("[response_A]\nvariant answer")
? "A"
: "B";
return Promise.resolve({
content: JSON.stringify({
verdict,
rationale: "variant is more accurate",
}),
endpoint: "mock",
message: { role: "assistant", content: "{}" },
provider: "cliproxy",
raw: {},
});
}
return Promise.resolve({
content: content.includes("사진 인증 위치를 명시해")
? "variant answer"
: "control answer",
endpoint: "mock",
message: { role: "assistant", content: "ok" },
provider: "cliproxy",
raw: {},
});
},
} as unknown as LlmClient;
const result = await runTournament(llmClient, {
cases: [archiveEntry("1-0"), archiveEntry("2-0")],
maxMatches: 4,
nowMs: Date.parse("2026-05-01T02:00:00.000Z"),
variantIds: ["control", "abc12345"],
variantManifests: {
abc12345: {
variantId: "abc12345",
description: "photo fix",
diff: [
{
layer: "policy",
op: "append",
text: "사진 인증 위치를 명시해.",
},
],
},
},
});
expect(result.generatedAt).toBe("2026-05-01T02:00:00.000Z");
expect(result.matches).toHaveLength(4);
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
"control",
"abc12345",
]);
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
"abc12345",
"control",
]);
expect(result.variants[0]).toMatchObject({
id: "abc12345",
matches: 4,
wins: 4,
});
expect(result.variants[0]?.elo).toBeGreaterThan(1_500);
});
});