This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 1453433b62
commit da76af3826
65 files changed
+11089 -143

No files matched your search

+363
View File
@@ -0,0 +1,363 @@
import { createHash } from "node:crypto";
import { z } from "zod";
import type { LlmClient } from "@llm/llm-client";
import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder";
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
import { getEvalTournamentMaxMatches } from "../../config.js";
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
import { getEvalArchiveCollection } from "../archive.js";
import {
createEloRating,
scoreForVerdict,
updateEloPair,
type EloRating,
} from "./elo.js";
export const EVAL_TOURNAMENT_COLLECTION = "eval_tournaments";
export interface TournamentMatch {
caseId: string;
judgeRationale: string;
judgeVerdict: "A" | "B" | "tie";
matchId: string;
responseA: string;
responseB: string;
variantA: string;
variantB: string;
}
export interface TournamentResult {
generatedAt: string;
matches: TournamentMatch[];
variants: EloRating[];
}
interface TournamentCollectionLike {
createIndex(
keys: Record<string, unknown>,
options?: Record<string, unknown>,
): Promise<unknown>;
find(filter: Record<string, unknown>): {
limit(value: number): {
sort(value: Record<string, 1 | -1>): {
toArray(): Promise<TournamentResult[]>;
};
};
sort(value: Record<string, 1 | -1>): {
limit(value: number): {
toArray(): Promise<TournamentResult[]>;
};
};
};
insertOne(document: TournamentResult): Promise<unknown>;
}
const sideBySideJudgeSchema = z.object({
rationale: z.string().trim().min(1).max(300),
verdict: z.enum(["A", "B", "tie"]),
});
function stableHash(value: unknown): string {
return createHash("sha256")
.update(JSON.stringify(value))
.digest("hex")
.slice(0, 12);
}
function emptyBlackboard(): Blackboard {
return {
flaggedIssues: [],
openQuestions: [],
recentNicknameAttempts: [],
toneMemory: {},
};
}
function promptContextForCase(entry: EvalArchiveEntry): PromptContext {
return {
blackboard: emptyBlackboard(),
conversationHistory: [],
userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.",
};
}
function getVariantManifest(
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): PromptVariantManifest | string | null {
if (variantId === "control") {
return null;
}
return manifests?.[variantId] ?? variantId;
}
async function generateTournamentResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string> {
let builder = PromptBuilder.forDerived(
entry.derived,
null,
promptContextForCase(entry),
);
const manifest = getVariantManifest(variantId, manifests);
if (manifest) {
builder = builder.applyVariant(manifest);
}
const response = await llmClient.chatCompletion(builder.build(), {
maxTokens: entry.derived.suggestedMaxTokens,
temperature: 0,
tier: entry.derived.tier,
});
return response?.content.trim() ?? "";
}
function buildJudgePrompt(
entry: EvalArchiveEntry,
responseA: string,
responseB: string,
): string {
return [
"아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.",
"평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.",
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
"",
"[request]",
JSON.stringify(entry.request),
"",
"[derived]",
JSON.stringify(entry.derived),
"",
"[archived_reference_response]",
entry.response.text,
"",
"[response_A]",
responseA,
"",
"[response_B]",
responseB,
"",
'JSON object only: {"verdict":"A","rationale":"..."}',
].join("\n");
}
async function judgeTournamentMatch(
llmClient: LlmClient,
entry: EvalArchiveEntry,
responseA: string,
responseB: string,
): Promise<{ rationale: string; verdict: "A" | "B" | "tie" }> {
const response = await llmClient.chatCompletion(
[
{
role: "system",
content:
"당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.",
},
{
role: "user",
content: buildJudgePrompt(entry, responseA, responseB),
},
],
{
jsonMode: true,
maxTokens: 256,
temperature: 0,
},
);
if (!response) {
return {
rationale: "judge-call-failed",
verdict: "tie",
};
}
try {
return sideBySideJudgeSchema.parse(JSON.parse(response.content));
} catch {
return {
rationale: "invalid-judge-json",
verdict: "tie",
};
}
}
function getTournamentCollection(
mongoConnection: MongoConnectionLike,
): TournamentCollectionLike {
return mongoConnection.collection(
EVAL_TOURNAMENT_COLLECTION,
) as TournamentCollectionLike;
}
async function ensureTournamentIndexes(
collection: TournamentCollectionLike,
): Promise<void> {
await Promise.all([
collection.createIndex({ generatedAt: -1 }),
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
]);
}
function buildVariantPairs(variantIds: string[]): Array<[string, string]> {
const pairs: Array<[string, string]> = [];
for (let left = 0; left < variantIds.length; left++) {
for (let right = left + 1; right < variantIds.length; right++) {
const variantA = variantIds[left];
const variantB = variantIds[right];
if (variantA && variantB) {
pairs.push([variantA, variantB], [variantB, variantA]);
}
}
}
return pairs;
}
export async function runTournament(
llmClient: LlmClient,
options: {
cases: EvalArchiveEntry[];
maxMatches?: number | undefined;
nowMs?: number | undefined;
variantIds: string[];
variantManifests?: Record<string, PromptVariantManifest> | undefined;
},
): Promise<TournamentResult> {
if (!llmClient.hasEndpoints()) {
throw new Error("Cannot run tournament without an LLM endpoint");
}
if (options.variantIds.length < 2) {
throw new Error("Tournament requires at least two variants");
}
const maxMatches = options.maxMatches ?? getEvalTournamentMaxMatches();
const ratings = new Map(
options.variantIds.map((variantId) => [variantId, createEloRating(variantId)]),
);
const matches: TournamentMatch[] = [];
const pairs = buildVariantPairs(options.variantIds);
for (const entry of options.cases) {
for (const [variantA, variantB] of pairs) {
if (matches.length >= maxMatches) {
break;
}
const [responseA, responseB] = await Promise.all([
generateTournamentResponse(
llmClient,
entry,
variantA,
options.variantManifests,
),
generateTournamentResponse(
llmClient,
entry,
variantB,
options.variantManifests,
),
]);
const judge = await judgeTournamentMatch(
llmClient,
entry,
responseA,
responseB,
);
const ratingA = ratings.get(variantA);
const ratingB = ratings.get(variantB);
if (!ratingA || !ratingB) {
throw new Error("Missing tournament rating");
}
const updated = updateEloPair(
ratingA,
ratingB,
scoreForVerdict(judge.verdict),
);
ratings.set(variantA, updated.ratingA);
ratings.set(variantB, updated.ratingB);
matches.push({
caseId: entry.streamId,
judgeRationale: judge.rationale,
judgeVerdict: judge.verdict,
matchId: stableHash({
caseId: entry.streamId,
index: matches.length,
variantA,
variantB,
}),
responseA,
responseB,
variantA,
variantB,
});
}
if (matches.length >= maxMatches) {
break;
}
}
return {
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
matches,
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
};
}
export async function loadTournamentCases(
mongoConnection: MongoConnectionLike,
options: {
caseIds?: string[] | undefined;
limit?: number | undefined;
},
): Promise<EvalArchiveEntry[]> {
const filter =
options.caseIds && options.caseIds.length > 0
? {
streamId: {
$in: options.caseIds,
},
}
: {};
return getEvalArchiveCollection(mongoConnection)
.find(filter)
.sort({ evaluatedAt: -1 })
.limit(options.limit ?? getEvalTournamentMaxMatches())
.toArray();
}
export async function saveTournamentResult(
mongoConnection: MongoConnectionLike,
result: TournamentResult,
): Promise<void> {
const collection = getTournamentCollection(mongoConnection);
await ensureTournamentIndexes(collection);
await collection.insertOne(result);
}
export async function queryTournamentResults(
mongoConnection: MongoConnectionLike,
options?: {
limit?: number | undefined;
variantId?: string | undefined;
},
): Promise<TournamentResult[]> {
const filter = options?.variantId
? {
"variants.id": options.variantId,
}
: {};
return getTournamentCollection(mongoConnection)
.find(filter)
.sort({ generatedAt: -1 })
.limit(options?.limit ?? 20)
.toArray();
}