import { createHash } from "node:crypto"; import { z } from "zod"; import type { LlmClient } from "@llm/llm-client"; import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder"; import type { Blackboard, PromptContext } from "@skills/onboarding/types"; import { getEvalTournamentMaxMatches } from "../../config.js"; import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js"; import { getEvalArchiveCollection } from "../archive.js"; import { createEloRating, scoreForVerdict, updateEloPair, type EloRating, } from "./elo.js"; export const EVAL_TOURNAMENT_COLLECTION = "eval_tournaments"; export interface TournamentMatch { caseId: string; judgeRationale: string; judgeVerdict: "A" | "B" | "tie"; matchId: string; responseA: string; responseB: string; variantA: string; variantB: string; } export interface TournamentResult { generatedAt: string; matches: TournamentMatch[]; variants: EloRating[]; } interface TournamentCollectionLike { createIndex( keys: Record, options?: Record, ): Promise; find(filter: Record): { limit(value: number): { sort(value: Record): { toArray(): Promise; }; }; sort(value: Record): { limit(value: number): { toArray(): Promise; }; }; }; insertOne(document: TournamentResult): Promise; } const sideBySideJudgeSchema = z.object({ rationale: z.string().trim().min(1).max(300), verdict: z.enum(["A", "B", "tie"]), }); function stableHash(value: unknown): string { return createHash("sha256") .update(JSON.stringify(value)) .digest("hex") .slice(0, 12); } function emptyBlackboard(): Blackboard { return { flaggedIssues: [], openQuestions: [], recentNicknameAttempts: [], toneMemory: {}, }; } function promptContextForCase(entry: EvalArchiveEntry): PromptContext { return { blackboard: emptyBlackboard(), conversationHistory: [], userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.", }; } function getVariantManifest( variantId: string, manifests?: Record, ): PromptVariantManifest | string | null { if (variantId === "control") { return null; } return manifests?.[variantId] ?? variantId; } async function generateTournamentResponse( llmClient: LlmClient, entry: EvalArchiveEntry, variantId: string, manifests?: Record, ): Promise { let builder = PromptBuilder.forDerived( entry.derived, null, promptContextForCase(entry), ); const manifest = getVariantManifest(variantId, manifests); if (manifest) { builder = builder.applyVariant(manifest); } const response = await llmClient.chatCompletion(builder.build(), { maxTokens: entry.derived.suggestedMaxTokens, temperature: 0, tier: entry.derived.tier, }); return response?.content.trim() ?? ""; } function buildJudgePrompt( entry: EvalArchiveEntry, responseA: string, responseB: string, ): string { return [ "아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.", "평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.", "A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.", "", "[request]", JSON.stringify(entry.request), "", "[derived]", JSON.stringify(entry.derived), "", "[archived_reference_response]", entry.response.text, "", "[response_A]", responseA, "", "[response_B]", responseB, "", 'JSON object only: {"verdict":"A","rationale":"..."}', ].join("\n"); } async function judgeTournamentMatch( llmClient: LlmClient, entry: EvalArchiveEntry, responseA: string, responseB: string, ): Promise<{ rationale: string; verdict: "A" | "B" | "tie" }> { const response = await llmClient.chatCompletion( [ { role: "system", content: "당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.", }, { role: "user", content: buildJudgePrompt(entry, responseA, responseB), }, ], { jsonMode: true, maxTokens: 256, temperature: 0, }, ); if (!response) { return { rationale: "judge-call-failed", verdict: "tie", }; } try { return sideBySideJudgeSchema.parse(JSON.parse(response.content)); } catch { return { rationale: "invalid-judge-json", verdict: "tie", }; } } function getTournamentCollection( mongoConnection: MongoConnectionLike, ): TournamentCollectionLike { return mongoConnection.collection( EVAL_TOURNAMENT_COLLECTION, ) as TournamentCollectionLike; } async function ensureTournamentIndexes( collection: TournamentCollectionLike, ): Promise { await Promise.all([ collection.createIndex({ generatedAt: -1 }), collection.createIndex({ "variants.id": 1, generatedAt: -1 }), ]); } function buildVariantPairs(variantIds: string[]): Array<[string, string]> { const pairs: Array<[string, string]> = []; for (let left = 0; left < variantIds.length; left++) { for (let right = left + 1; right < variantIds.length; right++) { const variantA = variantIds[left]; const variantB = variantIds[right]; if (variantA && variantB) { pairs.push([variantA, variantB], [variantB, variantA]); } } } return pairs; } export async function runTournament( llmClient: LlmClient, options: { cases: EvalArchiveEntry[]; maxMatches?: number | undefined; nowMs?: number | undefined; variantIds: string[]; variantManifests?: Record | undefined; }, ): Promise { if (!llmClient.hasEndpoints()) { throw new Error("Cannot run tournament without an LLM endpoint"); } if (options.variantIds.length < 2) { throw new Error("Tournament requires at least two variants"); } const maxMatches = options.maxMatches ?? getEvalTournamentMaxMatches(); const ratings = new Map( options.variantIds.map((variantId) => [variantId, createEloRating(variantId)]), ); const matches: TournamentMatch[] = []; const pairs = buildVariantPairs(options.variantIds); for (const entry of options.cases) { for (const [variantA, variantB] of pairs) { if (matches.length >= maxMatches) { break; } const [responseA, responseB] = await Promise.all([ generateTournamentResponse( llmClient, entry, variantA, options.variantManifests, ), generateTournamentResponse( llmClient, entry, variantB, options.variantManifests, ), ]); const judge = await judgeTournamentMatch( llmClient, entry, responseA, responseB, ); const ratingA = ratings.get(variantA); const ratingB = ratings.get(variantB); if (!ratingA || !ratingB) { throw new Error("Missing tournament rating"); } const updated = updateEloPair( ratingA, ratingB, scoreForVerdict(judge.verdict), ); ratings.set(variantA, updated.ratingA); ratings.set(variantB, updated.ratingB); matches.push({ caseId: entry.streamId, judgeRationale: judge.rationale, judgeVerdict: judge.verdict, matchId: stableHash({ caseId: entry.streamId, index: matches.length, variantA, variantB, }), responseA, responseB, variantA, variantB, }); } if (matches.length >= maxMatches) { break; } } return { generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(), matches, variants: [...ratings.values()].sort((left, right) => right.elo - left.elo), }; } export async function loadTournamentCases( mongoConnection: MongoConnectionLike, options: { caseIds?: string[] | undefined; limit?: number | undefined; }, ): Promise { const filter = options.caseIds && options.caseIds.length > 0 ? { streamId: { $in: options.caseIds, }, } : {}; return getEvalArchiveCollection(mongoConnection) .find(filter) .sort({ evaluatedAt: -1 }) .limit(options.limit ?? getEvalTournamentMaxMatches()) .toArray(); } export async function saveTournamentResult( mongoConnection: MongoConnectionLike, result: TournamentResult, ): Promise { const collection = getTournamentCollection(mongoConnection); await ensureTournamentIndexes(collection); await collection.insertOne(result); } export async function queryTournamentResults( mongoConnection: MongoConnectionLike, options?: { limit?: number | undefined; variantId?: string | undefined; }, ): Promise { const filter = options?.variantId ? { "variants.id": options.variantId, } : {}; return getTournamentCollection(mongoConnection) .find(filter) .sort({ generatedAt: -1 }) .limit(options?.limit ?? 20) .toArray(); }