140 lines
3.8 KiB
TypeScript
140 lines
3.8 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import type { LlmClient } from "@llm/llm-client";
|
|
import type { EvalArchiveEntry } from "../archive.js";
|
|
import { runTournament } from "./tournament.js";
|
|
|
|
function archiveEntry(streamId: string): EvalArchiveEntry {
|
|
return {
|
|
streamId,
|
|
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
|
request: {
|
|
requestId: streamId,
|
|
skill: "onboarding",
|
|
chatroomId: "room-1",
|
|
userId: "user-1",
|
|
nickname: "닉",
|
|
trigger: "user-message",
|
|
userMessage: "사진 어디로 보내요?",
|
|
},
|
|
response: {
|
|
requestId: streamId,
|
|
text: "이 채팅방에 올려주세요.",
|
|
confidence: 0.9,
|
|
source: "llm",
|
|
},
|
|
derived: {
|
|
phase: { kind: "photo-pending" },
|
|
intent: { kind: "photo-question" },
|
|
shouldRespond: true,
|
|
responseMode: "llm",
|
|
suggestedMaxTokens: 128,
|
|
allowShortResponse: true,
|
|
tier: 0,
|
|
},
|
|
classified: {
|
|
intent: { kind: "photo-question" },
|
|
confidence: 0.9,
|
|
source: "regex",
|
|
},
|
|
validation: {
|
|
status: "pass",
|
|
text: "이 채팅방에 올려주세요.",
|
|
reasons: [],
|
|
},
|
|
llmJudge: {
|
|
skipped: false,
|
|
model: "judge",
|
|
endpoint: "local",
|
|
provider: "test",
|
|
scores: {
|
|
friendliness: 4,
|
|
accuracy: 4,
|
|
concision: 4,
|
|
scope: 4,
|
|
overall: 4,
|
|
},
|
|
rationale: "ok",
|
|
raw: "{}",
|
|
},
|
|
latencyMs: 100,
|
|
tier: 0,
|
|
};
|
|
}
|
|
|
|
describe("runTournament", () => {
|
|
it("runs swapped A/B matches and updates ELO toward winning variants", async () => {
|
|
const llmClient = {
|
|
hasEndpoints: () => true,
|
|
chatCompletion: (
|
|
messages: Array<{ content: string; role: string }>,
|
|
options?: { jsonMode?: boolean | undefined },
|
|
) => {
|
|
const content = messages.map((message) => message.content).join("\n");
|
|
if (options?.jsonMode) {
|
|
const verdict = content.includes("[response_A]\nvariant answer")
|
|
? "A"
|
|
: "B";
|
|
return Promise.resolve({
|
|
content: JSON.stringify({
|
|
verdict,
|
|
rationale: "variant is more accurate",
|
|
}),
|
|
endpoint: "mock",
|
|
message: { role: "assistant", content: "{}" },
|
|
provider: "cliproxy",
|
|
raw: {},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: content.includes("사진 인증 위치를 명시해")
|
|
? "variant answer"
|
|
: "control answer",
|
|
endpoint: "mock",
|
|
message: { role: "assistant", content: "ok" },
|
|
provider: "cliproxy",
|
|
raw: {},
|
|
});
|
|
},
|
|
} as unknown as LlmClient;
|
|
|
|
const result = await runTournament(llmClient, {
|
|
cases: [archiveEntry("1-0"), archiveEntry("2-0")],
|
|
maxMatches: 4,
|
|
nowMs: Date.parse("2026-05-01T02:00:00.000Z"),
|
|
variantIds: ["control", "abc12345"],
|
|
variantManifests: {
|
|
abc12345: {
|
|
variantId: "abc12345",
|
|
description: "photo fix",
|
|
diff: [
|
|
{
|
|
layer: "policy",
|
|
op: "append",
|
|
text: "사진 인증 위치를 명시해.",
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
expect(result.generatedAt).toBe("2026-05-01T02:00:00.000Z");
|
|
expect(result.matches).toHaveLength(4);
|
|
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
|
|
"control",
|
|
"abc12345",
|
|
]);
|
|
expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([
|
|
"abc12345",
|
|
"control",
|
|
]);
|
|
expect(result.variants[0]).toMatchObject({
|
|
id: "abc12345",
|
|
matches: 4,
|
|
wins: 4,
|
|
});
|
|
expect(result.variants[0]?.elo).toBeGreaterThan(1_500);
|
|
});
|
|
});
|