import { describe, expect, it } from "vitest"; import type { LlmClient } from "@llm/llm-client"; import type { EvalArchiveEntry } from "../archive.js"; import { runTournament } from "./tournament.js"; function archiveEntry(streamId: string): EvalArchiveEntry { return { streamId, evaluatedAt: new Date("2026-05-01T00:00:00.000Z"), request: { requestId: streamId, skill: "onboarding", chatroomId: "room-1", userId: "user-1", nickname: "닉", trigger: "user-message", userMessage: "사진 어디로 보내요?", }, response: { requestId: streamId, text: "이 채팅방에 올려주세요.", confidence: 0.9, source: "llm", }, derived: { phase: { kind: "photo-pending" }, intent: { kind: "photo-question" }, shouldRespond: true, responseMode: "llm", suggestedMaxTokens: 128, allowShortResponse: true, tier: 0, }, classified: { intent: { kind: "photo-question" }, confidence: 0.9, source: "regex", }, validation: { status: "pass", text: "이 채팅방에 올려주세요.", reasons: [], }, llmJudge: { skipped: false, model: "judge", endpoint: "local", provider: "test", scores: { friendliness: 4, accuracy: 4, concision: 4, scope: 4, overall: 4, }, rationale: "ok", raw: "{}", }, latencyMs: 100, tier: 0, }; } describe("runTournament", () => { it("runs swapped A/B matches and updates ELO toward winning variants", async () => { const llmClient = { hasEndpoints: () => true, chatCompletion: ( messages: Array<{ content: string; role: string }>, options?: { jsonMode?: boolean | undefined }, ) => { const content = messages.map((message) => message.content).join("\n"); if (options?.jsonMode) { const verdict = content.includes("[response_A]\nvariant answer") ? "A" : "B"; return Promise.resolve({ content: JSON.stringify({ verdict, rationale: "variant is more accurate", }), endpoint: "mock", message: { role: "assistant", content: "{}" }, provider: "cliproxy", raw: {}, }); } return Promise.resolve({ content: content.includes("사진 인증 위치를 명시해") ? "variant answer" : "control answer", endpoint: "mock", message: { role: "assistant", content: "ok" }, provider: "cliproxy", raw: {}, }); }, } as unknown as LlmClient; const result = await runTournament(llmClient, { cases: [archiveEntry("1-0"), archiveEntry("2-0")], maxMatches: 4, nowMs: Date.parse("2026-05-01T02:00:00.000Z"), variantIds: ["control", "abc12345"], variantManifests: { abc12345: { variantId: "abc12345", description: "photo fix", diff: [ { layer: "policy", op: "append", text: "사진 인증 위치를 명시해.", }, ], }, }, }); expect(result.generatedAt).toBe("2026-05-01T02:00:00.000Z"); expect(result.matches).toHaveLength(4); expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([ "control", "abc12345", ]); expect(result.matches.map((match) => [match.variantA, match.variantB])).toContainEqual([ "abc12345", "control", ]); expect(result.variants[0]).toMatchObject({ id: "abc12345", matches: 4, wins: 4, }); expect(result.variants[0]?.elo).toBeGreaterThan(1_500); }); });