Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit a900016be4
60 files changed
+8689 -142

No files matched your search

+397
View File
@@ -1,3 +1,7 @@
import type { MongoConnectionLike } from "./archive.js";
import type { SliceKey, SliceReportRow } from "./aggregator.js";
import type { GarnetClient } from "@common/garnet";
function parsePositiveInteger(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
@@ -7,6 +11,78 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
function parsePositiveNumber(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
}
function parseDurationMs(value: string | undefined): number {
const raw = value?.trim().toLowerCase() || "7d";
const match = raw.match(/^(\d+)(h|d)$/u);
if (!match) {
throw new Error("--since must use a duration like 24h, 7d, or 30d");
}
const amount = Number(match[1]);
const unit = match[2];
if (!Number.isInteger(amount) || amount <= 0) {
throw new Error("--since must be a positive duration");
}
return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000;
}
function parseCsv(value: string | undefined): string[] {
if (!value?.trim()) {
return [];
}
return value
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
function parseSlice(value: string | undefined): SliceKey {
if (!value?.trim()) {
throw new Error(
"--slice must use tier=0,intent=status-question,phase=photo-pending",
);
}
const fields: Record<string, string> = {};
for (const pair of value.split(",")) {
const [key, ...rest] = pair.split("=");
const normalizedKey = key?.trim();
if (normalizedKey) {
fields[normalizedKey] = rest.join("=").trim();
}
}
const tier = Number(fields["tier"]);
const intent = fields["intent"];
const phase = fields["phase"];
if (
(tier !== 0 && tier !== 1 && tier !== 2) ||
!intent?.trim() ||
!phase?.trim()
) {
throw new Error(
"--slice must include tier=0|1|2,intent=<kind>,phase=<kind>",
);
}
return {
tier,
intent,
phase,
};
}
interface ParsedArgs {
command: string;
flags: Map<string, string | true>;
@@ -108,6 +184,122 @@ function printRunSummary(report: {
console.info(`[eval] report=${reportPath}`);
}
function formatNullable(value: number | null): string {
return value === null ? "n/a" : String(value);
}
function printSliceRows(rows: SliceReportRow[]): void {
console.info(
[
"tier",
"intent",
"phase",
"count",
"llmOverallAvg",
"validatorRejectRate",
"p50LatencyMs",
"p95LatencyMs",
"updatedAt",
].join("\t"),
);
for (const row of rows) {
console.info(
[
row.slice.tier,
row.slice.intent,
row.slice.phase,
row.count,
formatNullable(row.llmOverallAvg),
row.validatorRejectRate,
formatNullable(row.p50LatencyMs),
formatNullable(row.p95LatencyMs),
row.updatedAt,
].join("\t"),
);
}
}
function printCalibrationReport(report: {
generatedAt: string;
meanGapOverall: number;
pairs: number;
pearsonByDim: {
accuracy: number;
concision: number;
friendliness: number;
scope: number;
};
pearsonOverall: number;
worstDisagreements: Array<{
evalStreamId: string;
humanScore: number;
llmOverall: number;
}>;
}): void {
console.info(`[eval] generatedAt=${report.generatedAt}`);
console.info(`[eval] pairs=${report.pairs}`);
console.info(`[eval] pearsonOverall=${report.pearsonOverall}`);
console.info(`[eval] meanGapOverall=${report.meanGapOverall}`);
console.info(
`[eval] pearsonByDim friendliness=${report.pearsonByDim.friendliness} accuracy=${report.pearsonByDim.accuracy} concision=${report.pearsonByDim.concision} scope=${report.pearsonByDim.scope}`,
);
for (const item of report.worstDisagreements) {
console.info(
`[eval] disagreement evalStreamId=${item.evalStreamId} human=${item.humanScore} llm=${item.llmOverall}`,
);
}
}
function printTournamentResult(result: {
generatedAt: string;
matches: Array<{ judgeVerdict: string; variantA: string; variantB: string }>;
variants: Array<{ elo: number; id: string; matches: number; wins: number }>;
}): void {
console.info(`[eval] generatedAt=${result.generatedAt}`);
console.info(`[eval] matches=${result.matches.length}`);
for (const variant of result.variants) {
console.info(
`[eval] variant id=${variant.id} elo=${variant.elo} matches=${variant.matches} wins=${variant.wins}`,
);
}
}
async function connectMongoForEval(): Promise<{
close: () => Promise<void>;
connection: MongoConnectionLike;
}> {
const { initMongoDB, closeMongoDB } = await import("@common/mongodb");
const { getSecret } = await import("@common/secret-manager");
const connectionString =
getSecret("MONGODB_CONNECTION_STRING") ??
process.env["MONGODB_CONNECTION_STRING"];
if (!connectionString) {
throw new Error(
"Missing MONGODB_CONNECTION_STRING for eval command. Set it in the environment or secrets.",
);
}
return {
connection: await initMongoDB(connectionString),
close: closeMongoDB,
};
}
async function connectGarnetForEval(): Promise<{
close: () => Promise<void>;
garnetClient: GarnetClient;
}> {
const { GarnetClient } = await import("@common/garnet");
const garnetClient = new GarnetClient();
await garnetClient.connect();
return {
garnetClient,
close: () => garnetClient.close(),
};
}
async function main(): Promise<void> {
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
loadEnvironmentFromNodeEnv();
@@ -174,6 +366,211 @@ async function main(): Promise<void> {
return;
}
if (command === "report") {
const { queryArchiveSliceReportRows } = await import("./archive.js");
const limit =
parsePositiveInteger(getFlagString(flags, "worst")) ??
parsePositiveInteger(getFlagString(flags, "top")) ??
20;
const sort = hasFlag(flags, "worst") ? "score-asc" : "count-desc";
const mongo = await connectMongoForEval();
try {
const rows = await queryArchiveSliceReportRows(mongo.connection, {
limit,
sinceMs: parseDurationMs(getFlagString(flags, "since")),
sort,
});
printSliceRows(rows);
} finally {
await mongo.close();
}
return;
}
if (command === "promote") {
const { appendCandidatesToJsonl, selectPromotionCandidates } = await import(
"./promote.js"
);
const mongo = await connectMongoForEval();
try {
const candidates = await selectPromotionCandidates(mongo.connection, {
limit: parsePositiveInteger(getFlagString(flags, "limit")),
minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")),
});
if (hasFlag(flags, "dry-run")) {
for (const candidate of candidates) {
const overall = candidate.llmJudge.skipped
? "n/a"
: candidate.llmJudge.scores.overall;
console.info(
`[eval] candidate id=${candidate.caseId} reason=${candidate.reason} overall=${overall} trigger=${candidate.request.trigger}`,
);
}
console.info(`[eval] dryRunCandidates=${candidates.length}`);
return;
}
const result = await appendCandidatesToJsonl(
candidates,
getFlagString(flags, "output"),
);
console.info(
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
);
} finally {
await mongo.close();
}
return;
}
if (command === "review") {
const { parseReviewReason, runReviewCli } = await import("./review/cli.js");
const garnet = await connectGarnetForEval();
try {
const result = await runReviewCli(garnet.garnetClient, {
exportLabeled: getFlagString(flags, "export-labeled"),
limit: parsePositiveInteger(getFlagString(flags, "limit")),
reason: parseReviewReason(getFlagString(flags, "reason")),
resetPending: hasFlag(flags, "reset-pending"),
});
console.info(
`[eval] review completed labeled=${result.labeled} skipped=${result.skipped}`,
);
} finally {
await garnet.close();
}
return;
}
if (command === "calibrate") {
const { getEvalCalibrationMinCorr } = await import("../config.js");
const { computeCalibrationReport } = await import(
"./review/calibration.js"
);
const mongo = await connectMongoForEval();
const garnet = await connectGarnetForEval();
try {
const report = await computeCalibrationReport(
mongo.connection,
garnet.garnetClient,
{
sinceMs: getFlagString(flags, "since")
? parseDurationMs(getFlagString(flags, "since"))
: undefined,
},
);
printCalibrationReport(report);
if (report.pairs < 10) {
console.warn(
"[eval] calibration pairs<10; check gate is informational until at least 10 labels are available",
);
}
if (
hasFlag(flags, "check") &&
report.pairs >= 10 &&
report.pearsonOverall < getEvalCalibrationMinCorr()
) {
process.exitCode = 1;
}
} finally {
await garnet.close();
await mongo.close();
}
return;
}
if (command === "propose") {
const { createEvalLlmClient } = await import("./runner.js");
const {
buildProposalInputFromArchive,
proposeVariants,
writePromptVariantProposal,
} = await import("./loop/propose.js");
const mongo = await connectMongoForEval();
try {
const input = await buildProposalInputFromArchive(
mongo.connection,
parseSlice(getFlagString(flags, "slice")),
{
force: hasFlag(flags, "force"),
limit: parsePositiveInteger(getFlagString(flags, "limit")),
},
);
const proposals = await proposeVariants(createEvalLlmClient(), input);
for (const proposal of proposals.prompts) {
const outputPath = await writePromptVariantProposal(
proposal,
getFlagString(flags, "output-dir"),
);
console.info(
`[eval] proposal variant=${proposal.variantId} output=${outputPath}`,
);
}
for (const tool of proposals.tools) {
console.info(
`[eval] toolProposal variant=${tool.variantId} tool=${tool.toolName} why=${tool.whyNeeded}`,
);
}
} finally {
await mongo.close();
}
return;
}
if (command === "tournament") {
const { createEvalLlmClient } = await import("./runner.js");
const {
loadTournamentCases,
runTournament,
saveTournamentResult,
} = await import("./loop/tournament.js");
const variantIds = parseCsv(
getFlagString(flags, "variants") ?? positionals[0],
);
if (variantIds.length < 2) {
throw new Error(
'eval:tournament requires --variants=control,<variantId>',
);
}
const mongo = await connectMongoForEval();
try {
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
const cases = await loadTournamentCases(mongo.connection, {
caseIds: parseCsv(getFlagString(flags, "case-ids")),
limit: maxMatches,
});
const result = await runTournament(createEvalLlmClient(), {
cases,
maxMatches,
variantIds,
});
await saveTournamentResult(mongo.connection, result);
printTournamentResult(result);
} finally {
await mongo.close();
}
return;
}
if (command === "tournament-results") {
const { queryTournamentResults } = await import("./loop/tournament.js");
const mongo = await connectMongoForEval();
try {
const results = await queryTournamentResults(mongo.connection, {
limit: parsePositiveInteger(getFlagString(flags, "limit")),
variantId: getFlagString(flags, "variant"),
});
for (const result of results) {
printTournamentResult(result);
}
} finally {
await mongo.close();
}
return;
}
throw new Error(`Unknown eval command: ${command}`);
} catch (error) {
logger.error(