364 lines
9.2 KiB
TypeScript
364 lines
9.2 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
|
|
import { z } from "zod";
|
|
import type { LlmClient } from "@llm/llm-client";
|
|
import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
|
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
|
import { getEvalTournamentMaxMatches } from "../../config.js";
|
|
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
|
import { getEvalArchiveCollection } from "../archive.js";
|
|
import {
|
|
createEloRating,
|
|
scoreForVerdict,
|
|
updateEloPair,
|
|
type EloRating,
|
|
} from "./elo.js";
|
|
|
|
export const EVAL_TOURNAMENT_COLLECTION = "eval_tournaments";
|
|
|
|
export interface TournamentMatch {
|
|
caseId: string;
|
|
judgeRationale: string;
|
|
judgeVerdict: "A" | "B" | "tie";
|
|
matchId: string;
|
|
responseA: string;
|
|
responseB: string;
|
|
variantA: string;
|
|
variantB: string;
|
|
}
|
|
|
|
export interface TournamentResult {
|
|
generatedAt: string;
|
|
matches: TournamentMatch[];
|
|
variants: EloRating[];
|
|
}
|
|
|
|
interface TournamentCollectionLike {
|
|
createIndex(
|
|
keys: Record<string, unknown>,
|
|
options?: Record<string, unknown>,
|
|
): Promise<unknown>;
|
|
find(filter: Record<string, unknown>): {
|
|
limit(value: number): {
|
|
sort(value: Record<string, 1 | -1>): {
|
|
toArray(): Promise<TournamentResult[]>;
|
|
};
|
|
};
|
|
sort(value: Record<string, 1 | -1>): {
|
|
limit(value: number): {
|
|
toArray(): Promise<TournamentResult[]>;
|
|
};
|
|
};
|
|
};
|
|
insertOne(document: TournamentResult): Promise<unknown>;
|
|
}
|
|
|
|
const sideBySideJudgeSchema = z.object({
|
|
rationale: z.string().trim().min(1).max(300),
|
|
verdict: z.enum(["A", "B", "tie"]),
|
|
});
|
|
|
|
function stableHash(value: unknown): string {
|
|
return createHash("sha256")
|
|
.update(JSON.stringify(value))
|
|
.digest("hex")
|
|
.slice(0, 12);
|
|
}
|
|
|
|
function emptyBlackboard(): Blackboard {
|
|
return {
|
|
flaggedIssues: [],
|
|
openQuestions: [],
|
|
recentNicknameAttempts: [],
|
|
toneMemory: {},
|
|
};
|
|
}
|
|
|
|
function promptContextForCase(entry: EvalArchiveEntry): PromptContext {
|
|
return {
|
|
blackboard: emptyBlackboard(),
|
|
conversationHistory: [],
|
|
userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.",
|
|
};
|
|
}
|
|
|
|
function getVariantManifest(
|
|
variantId: string,
|
|
manifests?: Record<string, PromptVariantManifest>,
|
|
): PromptVariantManifest | string | null {
|
|
if (variantId === "control") {
|
|
return null;
|
|
}
|
|
return manifests?.[variantId] ?? variantId;
|
|
}
|
|
|
|
async function generateTournamentResponse(
|
|
llmClient: LlmClient,
|
|
entry: EvalArchiveEntry,
|
|
variantId: string,
|
|
manifests?: Record<string, PromptVariantManifest>,
|
|
): Promise<string> {
|
|
let builder = PromptBuilder.forDerived(
|
|
entry.derived,
|
|
null,
|
|
promptContextForCase(entry),
|
|
);
|
|
const manifest = getVariantManifest(variantId, manifests);
|
|
if (manifest) {
|
|
builder = builder.applyVariant(manifest);
|
|
}
|
|
|
|
const response = await llmClient.chatCompletion(builder.build(), {
|
|
maxTokens: entry.derived.suggestedMaxTokens,
|
|
temperature: 0,
|
|
tier: entry.derived.tier,
|
|
});
|
|
|
|
return response?.content.trim() ?? "";
|
|
}
|
|
|
|
function buildJudgePrompt(
|
|
entry: EvalArchiveEntry,
|
|
responseA: string,
|
|
responseB: string,
|
|
): string {
|
|
return [
|
|
"아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.",
|
|
"평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.",
|
|
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
|
|
"",
|
|
"[request]",
|
|
JSON.stringify(entry.request),
|
|
"",
|
|
"[derived]",
|
|
JSON.stringify(entry.derived),
|
|
"",
|
|
"[archived_reference_response]",
|
|
entry.response.text,
|
|
"",
|
|
"[response_A]",
|
|
responseA,
|
|
"",
|
|
"[response_B]",
|
|
responseB,
|
|
"",
|
|
'JSON object only: {"verdict":"A","rationale":"..."}',
|
|
].join("\n");
|
|
}
|
|
|
|
async function judgeTournamentMatch(
|
|
llmClient: LlmClient,
|
|
entry: EvalArchiveEntry,
|
|
responseA: string,
|
|
responseB: string,
|
|
): Promise<{ rationale: string; verdict: "A" | "B" | "tie" }> {
|
|
const response = await llmClient.chatCompletion(
|
|
[
|
|
{
|
|
role: "system",
|
|
content:
|
|
"당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.",
|
|
},
|
|
{
|
|
role: "user",
|
|
content: buildJudgePrompt(entry, responseA, responseB),
|
|
},
|
|
],
|
|
{
|
|
jsonMode: true,
|
|
maxTokens: 256,
|
|
temperature: 0,
|
|
},
|
|
);
|
|
|
|
if (!response) {
|
|
return {
|
|
rationale: "judge-call-failed",
|
|
verdict: "tie",
|
|
};
|
|
}
|
|
|
|
try {
|
|
return sideBySideJudgeSchema.parse(JSON.parse(response.content));
|
|
} catch {
|
|
return {
|
|
rationale: "invalid-judge-json",
|
|
verdict: "tie",
|
|
};
|
|
}
|
|
}
|
|
|
|
function getTournamentCollection(
|
|
mongoConnection: MongoConnectionLike,
|
|
): TournamentCollectionLike {
|
|
return mongoConnection.collection(
|
|
EVAL_TOURNAMENT_COLLECTION,
|
|
) as TournamentCollectionLike;
|
|
}
|
|
|
|
async function ensureTournamentIndexes(
|
|
collection: TournamentCollectionLike,
|
|
): Promise<void> {
|
|
await Promise.all([
|
|
collection.createIndex({ generatedAt: -1 }),
|
|
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
|
|
]);
|
|
}
|
|
|
|
function buildVariantPairs(variantIds: string[]): Array<[string, string]> {
|
|
const pairs: Array<[string, string]> = [];
|
|
for (let left = 0; left < variantIds.length; left++) {
|
|
for (let right = left + 1; right < variantIds.length; right++) {
|
|
const variantA = variantIds[left];
|
|
const variantB = variantIds[right];
|
|
if (variantA && variantB) {
|
|
pairs.push([variantA, variantB], [variantB, variantA]);
|
|
}
|
|
}
|
|
}
|
|
return pairs;
|
|
}
|
|
|
|
export async function runTournament(
|
|
llmClient: LlmClient,
|
|
options: {
|
|
cases: EvalArchiveEntry[];
|
|
maxMatches?: number | undefined;
|
|
nowMs?: number | undefined;
|
|
variantIds: string[];
|
|
variantManifests?: Record<string, PromptVariantManifest> | undefined;
|
|
},
|
|
): Promise<TournamentResult> {
|
|
if (!llmClient.hasEndpoints()) {
|
|
throw new Error("Cannot run tournament without an LLM endpoint");
|
|
}
|
|
if (options.variantIds.length < 2) {
|
|
throw new Error("Tournament requires at least two variants");
|
|
}
|
|
|
|
const maxMatches = options.maxMatches ?? getEvalTournamentMaxMatches();
|
|
const ratings = new Map(
|
|
options.variantIds.map((variantId) => [variantId, createEloRating(variantId)]),
|
|
);
|
|
const matches: TournamentMatch[] = [];
|
|
const pairs = buildVariantPairs(options.variantIds);
|
|
|
|
for (const entry of options.cases) {
|
|
for (const [variantA, variantB] of pairs) {
|
|
if (matches.length >= maxMatches) {
|
|
break;
|
|
}
|
|
|
|
const [responseA, responseB] = await Promise.all([
|
|
generateTournamentResponse(
|
|
llmClient,
|
|
entry,
|
|
variantA,
|
|
options.variantManifests,
|
|
),
|
|
generateTournamentResponse(
|
|
llmClient,
|
|
entry,
|
|
variantB,
|
|
options.variantManifests,
|
|
),
|
|
]);
|
|
const judge = await judgeTournamentMatch(
|
|
llmClient,
|
|
entry,
|
|
responseA,
|
|
responseB,
|
|
);
|
|
const ratingA = ratings.get(variantA);
|
|
const ratingB = ratings.get(variantB);
|
|
if (!ratingA || !ratingB) {
|
|
throw new Error("Missing tournament rating");
|
|
}
|
|
const updated = updateEloPair(
|
|
ratingA,
|
|
ratingB,
|
|
scoreForVerdict(judge.verdict),
|
|
);
|
|
ratings.set(variantA, updated.ratingA);
|
|
ratings.set(variantB, updated.ratingB);
|
|
|
|
matches.push({
|
|
caseId: entry.streamId,
|
|
judgeRationale: judge.rationale,
|
|
judgeVerdict: judge.verdict,
|
|
matchId: stableHash({
|
|
caseId: entry.streamId,
|
|
index: matches.length,
|
|
variantA,
|
|
variantB,
|
|
}),
|
|
responseA,
|
|
responseB,
|
|
variantA,
|
|
variantB,
|
|
});
|
|
}
|
|
if (matches.length >= maxMatches) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return {
|
|
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
|
|
matches,
|
|
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
|
|
};
|
|
}
|
|
|
|
export async function loadTournamentCases(
|
|
mongoConnection: MongoConnectionLike,
|
|
options: {
|
|
caseIds?: string[] | undefined;
|
|
limit?: number | undefined;
|
|
},
|
|
): Promise<EvalArchiveEntry[]> {
|
|
const filter =
|
|
options.caseIds && options.caseIds.length > 0
|
|
? {
|
|
streamId: {
|
|
$in: options.caseIds,
|
|
},
|
|
}
|
|
: {};
|
|
|
|
return getEvalArchiveCollection(mongoConnection)
|
|
.find(filter)
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(options.limit ?? getEvalTournamentMaxMatches())
|
|
.toArray();
|
|
}
|
|
|
|
export async function saveTournamentResult(
|
|
mongoConnection: MongoConnectionLike,
|
|
result: TournamentResult,
|
|
): Promise<void> {
|
|
const collection = getTournamentCollection(mongoConnection);
|
|
await ensureTournamentIndexes(collection);
|
|
await collection.insertOne(result);
|
|
}
|
|
|
|
export async function queryTournamentResults(
|
|
mongoConnection: MongoConnectionLike,
|
|
options?: {
|
|
limit?: number | undefined;
|
|
variantId?: string | undefined;
|
|
},
|
|
): Promise<TournamentResult[]> {
|
|
const filter = options?.variantId
|
|
? {
|
|
"variants.id": options.variantId,
|
|
}
|
|
: {};
|
|
|
|
return getTournamentCollection(mongoConnection)
|
|
.find(filter)
|
|
.sort({ generatedAt: -1 })
|
|
.limit(options?.limit ?? 20)
|
|
.toArray();
|
|
}
|