Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -0,0 +1,139 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { EvalArchiveEntry } from "../archive.js";
|
||||
import { runTournament } from "./tournament.js";
|
||||
|
||||
function archiveEntry(streamId: string): EvalArchiveEntry {
|
||||
return {
|
||||
streamId,
|
||||
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
||||
request: {
|
||||
requestId: streamId,
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
userMessage: "사진 어디로 보내요?",
|
||||
},
|
||||
response: {
|
||||
requestId: streamId,
|
||||
text: "이 채팅방에 올려주세요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "photo-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "photo-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "이 채팅방에 올려주세요.",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: 4,
|
||||
accuracy: 4,
|
||||
concision: 4,
|
||||
scope: 4,
|
||||
overall: 4,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
},
|
||||
latencyMs: 100,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("runTournament", () => {
|
||||
it("runs swapped A/B matches and updates ELO toward winning variants", async () => {
|
||||
const llmClient = {
|
||||
hasEndpoints: () => true,
|
||||
chatCompletion: (
|
||||
messages: Array<{ content: string; role: string }>,
|
||||
options?: { jsonMode?: boolean | undefined },
|
||||
) => {
|
||||
const content = messages.map((message) => message.content).join("\n");
|
||||
if (options?.jsonMode) {
|
||||
const verdict = content.includes("[response_A]\nvariant answer")
|
||||
? "A"
|
||||
: "B";
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
verdict,
|
||||
rationale: "variant is more accurate",
|
||||
}),
|
||||
endpoint: "mock",
|
||||
message: { role: "assistant", content: "{}" },
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: content.includes("사진 인증 위치를 명시해")
|
||||
? "variant answer"
|
||||
: "control answer",
|
||||
endpoint: "mock",
|
||||
message: { role: "assistant", content: "ok" },
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
});
|
||||
},
|
||||
} as unknown as LlmClient;
|
||||
|
||||
const result = await runTournament(llmClient, {
|
||||
cases: [archiveEntry("1-0"), archiveEntry("2-0")],
|
||||
maxMatches: 4,
|
||||
nowMs: Date.parse("2026-05-01T02:00:00.000Z"),
|
||||
variantIds: ["control", "abc12345"],
|
||||
variantManifests: {
|
||||
abc12345: {
|
||||
variantId: "abc12345",
|
||||
description: "photo fix",
|
||||
diff: [
|
||||
{
|
||||
layer: "policy",
|
||||
op: "append",
|
||||
text: "사진 인증 위치를 명시해.",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.generatedAt).toBe("2026-05-01T02:00:00.000Z");
|
||||
expect(result.matches).toHaveLength(4);
|
||||
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
|
||||
"control",
|
||||
"abc12345",
|
||||
]);
|
||||
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
|
||||
"abc12345",
|
||||
"control",
|
||||
]);
|
||||
expect(result.variants[0]).toMatchObject({
|
||||
id: "abc12345",
|
||||
matches: 4,
|
||||
wins: 4,
|
||||
});
|
||||
expect(result.variants[0]?.elo).toBeGreaterThan(1_500);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user