.
This commit is contained in:
1 parent
1453433b62
commit
da76af3826
65 files changed
+11089
-143
No files matched your search
@@ -0,0 +1,363 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { PromptBuilder, type PromptVariantManifest } from "@skills/onboarding/prompts/builder";
|
||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
||||
import { getEvalTournamentMaxMatches } from "../../config.js";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
||||
import { getEvalArchiveCollection } from "../archive.js";
|
||||
import {
|
||||
createEloRating,
|
||||
scoreForVerdict,
|
||||
updateEloPair,
|
||||
type EloRating,
|
||||
} from "./elo.js";
|
||||
|
||||
export const EVAL_TOURNAMENT_COLLECTION = "eval_tournaments";
|
||||
|
||||
export interface TournamentMatch {
|
||||
caseId: string;
|
||||
judgeRationale: string;
|
||||
judgeVerdict: "A" | "B" | "tie";
|
||||
matchId: string;
|
||||
responseA: string;
|
||||
responseB: string;
|
||||
variantA: string;
|
||||
variantB: string;
|
||||
}
|
||||
|
||||
export interface TournamentResult {
|
||||
generatedAt: string;
|
||||
matches: TournamentMatch[];
|
||||
variants: EloRating[];
|
||||
}
|
||||
|
||||
interface TournamentCollectionLike {
|
||||
createIndex(
|
||||
keys: Record<string, unknown>,
|
||||
options?: Record<string, unknown>,
|
||||
): Promise<unknown>;
|
||||
find(filter: Record<string, unknown>): {
|
||||
limit(value: number): {
|
||||
sort(value: Record<string, 1 | -1>): {
|
||||
toArray(): Promise<TournamentResult[]>;
|
||||
};
|
||||
};
|
||||
sort(value: Record<string, 1 | -1>): {
|
||||
limit(value: number): {
|
||||
toArray(): Promise<TournamentResult[]>;
|
||||
};
|
||||
};
|
||||
};
|
||||
insertOne(document: TournamentResult): Promise<unknown>;
|
||||
}
|
||||
|
||||
const sideBySideJudgeSchema = z.object({
|
||||
rationale: z.string().trim().min(1).max(300),
|
||||
verdict: z.enum(["A", "B", "tie"]),
|
||||
});
|
||||
|
||||
function stableHash(value: unknown): string {
|
||||
return createHash("sha256")
|
||||
.update(JSON.stringify(value))
|
||||
.digest("hex")
|
||||
.slice(0, 12);
|
||||
}
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function promptContextForCase(entry: EvalArchiveEntry): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage: entry.request.userMessage?.trim() || "필요한 온보딩 안내만 답해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
function getVariantManifest(
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): PromptVariantManifest | string | null {
|
||||
if (variantId === "control") {
|
||||
return null;
|
||||
}
|
||||
return manifests?.[variantId] ?? variantId;
|
||||
}
|
||||
|
||||
async function generateTournamentResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string> {
|
||||
let builder = PromptBuilder.forDerived(
|
||||
entry.derived,
|
||||
null,
|
||||
promptContextForCase(entry),
|
||||
);
|
||||
const manifest = getVariantManifest(variantId, manifests);
|
||||
if (manifest) {
|
||||
builder = builder.applyVariant(manifest);
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(builder.build(), {
|
||||
maxTokens: entry.derived.suggestedMaxTokens,
|
||||
temperature: 0,
|
||||
tier: entry.derived.tier,
|
||||
});
|
||||
|
||||
return response?.content.trim() ?? "";
|
||||
}
|
||||
|
||||
function buildJudgePrompt(
|
||||
entry: EvalArchiveEntry,
|
||||
responseA: string,
|
||||
responseB: string,
|
||||
): string {
|
||||
return [
|
||||
"아래 두 온보딩 응답 중 더 나은 쪽을 고르세요.",
|
||||
"평가 기준은 정확성, 질문 범위 준수, 친근함, 간결함입니다.",
|
||||
"A/B 위치 편향 없이 판단하고, 둘 다 비슷하면 tie를 선택하세요.",
|
||||
"",
|
||||
"[request]",
|
||||
JSON.stringify(entry.request),
|
||||
"",
|
||||
"[derived]",
|
||||
JSON.stringify(entry.derived),
|
||||
"",
|
||||
"[archived_reference_response]",
|
||||
entry.response.text,
|
||||
"",
|
||||
"[response_A]",
|
||||
responseA,
|
||||
"",
|
||||
"[response_B]",
|
||||
responseB,
|
||||
"",
|
||||
'JSON object only: {"verdict":"A","rationale":"..."}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function judgeTournamentMatch(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
responseA: string,
|
||||
responseB: string,
|
||||
): Promise<{ rationale: string; verdict: "A" | "B" | "tie" }> {
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 응답 A/B 테스트 심판입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildJudgePrompt(entry, responseA, responseB),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: 256,
|
||||
temperature: 0,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
return {
|
||||
rationale: "judge-call-failed",
|
||||
verdict: "tie",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return sideBySideJudgeSchema.parse(JSON.parse(response.content));
|
||||
} catch {
|
||||
return {
|
||||
rationale: "invalid-judge-json",
|
||||
verdict: "tie",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function getTournamentCollection(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
): TournamentCollectionLike {
|
||||
return mongoConnection.collection(
|
||||
EVAL_TOURNAMENT_COLLECTION,
|
||||
) as TournamentCollectionLike;
|
||||
}
|
||||
|
||||
async function ensureTournamentIndexes(
|
||||
collection: TournamentCollectionLike,
|
||||
): Promise<void> {
|
||||
await Promise.all([
|
||||
collection.createIndex({ generatedAt: -1 }),
|
||||
collection.createIndex({ "variants.id": 1, generatedAt: -1 }),
|
||||
]);
|
||||
}
|
||||
|
||||
function buildVariantPairs(variantIds: string[]): Array<[string, string]> {
|
||||
const pairs: Array<[string, string]> = [];
|
||||
for (let left = 0; left < variantIds.length; left++) {
|
||||
for (let right = left + 1; right < variantIds.length; right++) {
|
||||
const variantA = variantIds[left];
|
||||
const variantB = variantIds[right];
|
||||
if (variantA && variantB) {
|
||||
pairs.push([variantA, variantB], [variantB, variantA]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
export async function runTournament(
|
||||
llmClient: LlmClient,
|
||||
options: {
|
||||
cases: EvalArchiveEntry[];
|
||||
maxMatches?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
variantIds: string[];
|
||||
variantManifests?: Record<string, PromptVariantManifest> | undefined;
|
||||
},
|
||||
): Promise<TournamentResult> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
throw new Error("Cannot run tournament without an LLM endpoint");
|
||||
}
|
||||
if (options.variantIds.length < 2) {
|
||||
throw new Error("Tournament requires at least two variants");
|
||||
}
|
||||
|
||||
const maxMatches = options.maxMatches ?? getEvalTournamentMaxMatches();
|
||||
const ratings = new Map(
|
||||
options.variantIds.map((variantId) => [variantId, createEloRating(variantId)]),
|
||||
);
|
||||
const matches: TournamentMatch[] = [];
|
||||
const pairs = buildVariantPairs(options.variantIds);
|
||||
|
||||
for (const entry of options.cases) {
|
||||
for (const [variantA, variantB] of pairs) {
|
||||
if (matches.length >= maxMatches) {
|
||||
break;
|
||||
}
|
||||
|
||||
const [responseA, responseB] = await Promise.all([
|
||||
generateTournamentResponse(
|
||||
llmClient,
|
||||
entry,
|
||||
variantA,
|
||||
options.variantManifests,
|
||||
),
|
||||
generateTournamentResponse(
|
||||
llmClient,
|
||||
entry,
|
||||
variantB,
|
||||
options.variantManifests,
|
||||
),
|
||||
]);
|
||||
const judge = await judgeTournamentMatch(
|
||||
llmClient,
|
||||
entry,
|
||||
responseA,
|
||||
responseB,
|
||||
);
|
||||
const ratingA = ratings.get(variantA);
|
||||
const ratingB = ratings.get(variantB);
|
||||
if (!ratingA || !ratingB) {
|
||||
throw new Error("Missing tournament rating");
|
||||
}
|
||||
const updated = updateEloPair(
|
||||
ratingA,
|
||||
ratingB,
|
||||
scoreForVerdict(judge.verdict),
|
||||
);
|
||||
ratings.set(variantA, updated.ratingA);
|
||||
ratings.set(variantB, updated.ratingB);
|
||||
|
||||
matches.push({
|
||||
caseId: entry.streamId,
|
||||
judgeRationale: judge.rationale,
|
||||
judgeVerdict: judge.verdict,
|
||||
matchId: stableHash({
|
||||
caseId: entry.streamId,
|
||||
index: matches.length,
|
||||
variantA,
|
||||
variantB,
|
||||
}),
|
||||
responseA,
|
||||
responseB,
|
||||
variantA,
|
||||
variantB,
|
||||
});
|
||||
}
|
||||
if (matches.length >= maxMatches) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
generatedAt: new Date(options.nowMs ?? Date.now()).toISOString(),
|
||||
matches,
|
||||
variants: [...ratings.values()].sort((left, right) => right.elo - left.elo),
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadTournamentCases(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options: {
|
||||
caseIds?: string[] | undefined;
|
||||
limit?: number | undefined;
|
||||
},
|
||||
): Promise<EvalArchiveEntry[]> {
|
||||
const filter =
|
||||
options.caseIds && options.caseIds.length > 0
|
||||
? {
|
||||
streamId: {
|
||||
$in: options.caseIds,
|
||||
},
|
||||
}
|
||||
: {};
|
||||
|
||||
return getEvalArchiveCollection(mongoConnection)
|
||||
.find(filter)
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(options.limit ?? getEvalTournamentMaxMatches())
|
||||
.toArray();
|
||||
}
|
||||
|
||||
export async function saveTournamentResult(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
result: TournamentResult,
|
||||
): Promise<void> {
|
||||
const collection = getTournamentCollection(mongoConnection);
|
||||
await ensureTournamentIndexes(collection);
|
||||
await collection.insertOne(result);
|
||||
}
|
||||
|
||||
export async function queryTournamentResults(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
limit?: number | undefined;
|
||||
variantId?: string | undefined;
|
||||
},
|
||||
): Promise<TournamentResult[]> {
|
||||
const filter = options?.variantId
|
||||
? {
|
||||
"variants.id": options.variantId,
|
||||
}
|
||||
: {};
|
||||
|
||||
return getTournamentCollection(mongoConnection)
|
||||
.find(filter)
|
||||
.sort({ generatedAt: -1 })
|
||||
.limit(options?.limit ?? 20)
|
||||
.toArray();
|
||||
}
|
||||
Reference in new issue
Block a user