import type { MongoConnectionLike } from "./archive.js"; import type { SliceKey, SliceReportRow } from "./aggregator.js"; import type { GarnetClient } from "@common/garnet"; function parsePositiveInteger(value: string | undefined): number | undefined { if (!value?.trim()) { return undefined; } const parsed = Number(value); return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; } function parsePositiveNumber(value: string | undefined): number | undefined { if (!value?.trim()) { return undefined; } const parsed = Number(value); return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; } function parseDurationMs(value: string | undefined): number { const raw = value?.trim().toLowerCase() || "7d"; const match = raw.match(/^(\d+)(h|d)$/u); if (!match) { throw new Error("--since must use a duration like 24h, 7d, or 30d"); } const amount = Number(match[1]); const unit = match[2]; if (!Number.isInteger(amount) || amount <= 0) { throw new Error("--since must be a positive duration"); } return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000; } function parseCsv(value: string | undefined): string[] { if (!value?.trim()) { return []; } return value .split(",") .map((item) => item.trim()) .filter((item) => item.length > 0); } function parseSlice(value: string | undefined): SliceKey { if (!value?.trim()) { throw new Error( "--slice must use tier=0,intent=status-question,phase=photo-pending", ); } const fields: Record = {}; for (const pair of value.split(",")) { const [key, ...rest] = pair.split("="); const normalizedKey = key?.trim(); if (normalizedKey) { fields[normalizedKey] = rest.join("=").trim(); } } const tier = Number(fields["tier"]); const intent = fields["intent"]; const phase = fields["phase"]; if ( (tier !== 0 && tier !== 1 && tier !== 2) || !intent?.trim() || !phase?.trim() ) { throw new Error( "--slice must include tier=0|1|2,intent=,phase=", ); } return { tier, intent, phase, }; } interface ParsedArgs { command: string; flags: Map; positionals: string[]; } function parseArgs(argv: string[]): ParsedArgs { const [command = "run", ...rest] = argv; const flags = new Map(); const positionals: string[] = []; for (let index = 0; index < rest.length; index++) { const token = rest[index]; if (!token) { continue; } if (!token.startsWith("--")) { positionals.push(token); continue; } const withoutPrefix = token.slice(2); const equalsIndex = withoutPrefix.indexOf("="); if (equalsIndex >= 0) { const key = withoutPrefix.slice(0, equalsIndex); const value = withoutPrefix.slice(equalsIndex + 1); flags.set(key, value || true); continue; } const next = rest[index + 1]; if (next && !next.startsWith("--")) { flags.set(withoutPrefix, next); index += 1; continue; } flags.set(withoutPrefix, true); } return { command, flags, positionals, }; } function getFlagString( flags: Map, key: string, ): string | undefined { const value = flags.get(key); return typeof value === "string" ? value : undefined; } function hasFlag( flags: Map, key: string, ): boolean { return flags.has(key); } function printRunSummary(report: { generatedAt: string; summary: { cases: number; passRate: number; ruleAverageScore: number; averageLatencyMs: number; llmAverageOverall: number | null; llmJudgedCases: number; }; baselineDelta?: { passRate: number | null; ruleAverageScore: number | null; llmAverageOverall: number | null; averageLatencyMs: number | null; } | undefined; }, reportPath: string): void { const llmLine = report.summary.llmAverageOverall === null ? "n/a" : `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`; console.info(`[eval] generatedAt=${report.generatedAt}`); console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`); console.info( `[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`, ); console.info(`[eval] llmAverageOverall=${llmLine}`); if (report.baselineDelta) { console.info( `[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`, ); } console.info(`[eval] report=${reportPath}`); } function formatNullable(value: number | null): string { return value === null ? "n/a" : String(value); } function printSliceRows(rows: SliceReportRow[]): void { console.info( [ "tier", "intent", "phase", "count", "llmOverallAvg", "validatorRejectRate", "p50LatencyMs", "p95LatencyMs", "updatedAt", ].join("\t"), ); for (const row of rows) { console.info( [ row.slice.tier, row.slice.intent, row.slice.phase, row.count, formatNullable(row.llmOverallAvg), row.validatorRejectRate, formatNullable(row.p50LatencyMs), formatNullable(row.p95LatencyMs), row.updatedAt, ].join("\t"), ); } } function printCalibrationReport(report: { generatedAt: string; meanGapOverall: number; pairs: number; pearsonByDim: { accuracy: number; concision: number; friendliness: number; scope: number; }; pearsonOverall: number; worstDisagreements: Array<{ evalStreamId: string; humanScore: number; llmOverall: number; }>; }): void { console.info(`[eval] generatedAt=${report.generatedAt}`); console.info(`[eval] pairs=${report.pairs}`); console.info(`[eval] pearsonOverall=${report.pearsonOverall}`); console.info(`[eval] meanGapOverall=${report.meanGapOverall}`); console.info( `[eval] pearsonByDim friendliness=${report.pearsonByDim.friendliness} accuracy=${report.pearsonByDim.accuracy} concision=${report.pearsonByDim.concision} scope=${report.pearsonByDim.scope}`, ); for (const item of report.worstDisagreements) { console.info( `[eval] disagreement evalStreamId=${item.evalStreamId} human=${item.humanScore} llm=${item.llmOverall}`, ); } } function printTournamentResult(result: { generatedAt: string; matches: Array<{ judgeVerdict: string; variantA: string; variantB: string }>; variants: Array<{ elo: number; id: string; matches: number; wins: number }>; }): void { console.info(`[eval] generatedAt=${result.generatedAt}`); console.info(`[eval] matches=${result.matches.length}`); for (const variant of result.variants) { console.info( `[eval] variant id=${variant.id} elo=${variant.elo} matches=${variant.matches} wins=${variant.wins}`, ); } } async function connectMongoForEval(): Promise<{ close: () => Promise; connection: MongoConnectionLike; }> { const { initMongoDB, closeMongoDB } = await import("@common/mongodb"); const { getSecret } = await import("@common/secret-manager"); const connectionString = getSecret("MONGODB_CONNECTION_STRING") ?? process.env["MONGODB_CONNECTION_STRING"]; if (!connectionString) { throw new Error( "Missing MONGODB_CONNECTION_STRING for eval command. Set it in the environment or secrets.", ); } return { connection: await initMongoDB(connectionString), close: closeMongoDB, }; } async function connectGarnetForEval(): Promise<{ close: () => Promise; garnetClient: GarnetClient; }> { const { GarnetClient } = await import("@common/garnet"); const garnetClient = new GarnetClient(); await garnetClient.connect(); return { garnetClient, close: () => garnetClient.close(), }; } async function main(): Promise { const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader"); loadEnvironmentFromNodeEnv(); const { getLogger, initLogger } = await import("@obs/logger"); initLogger(); const logger = getLogger(); const parsed = parseArgs(process.argv.slice(2)); const { command, flags, positionals } = parsed; try { if (command === "capture") { const { captureGoldenSetScaffold } = await import("./capture.js"); const outputPath = await captureGoldenSetScaffold({ limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? parsePositiveInteger(positionals[0]), outputPath: getFlagString(flags, "output") ?? positionals[1], }); logger.info({ outputPath }, '[eval] "capture" completed'); return; } if (command === "run") { const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import( "./runner.js" ); const { publishEvalReportIfPossible } = await import( "./runner.js" ); const setName = getFlagString(flags, "set") ?? "golden"; const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName); const limit = parsePositiveInteger(getFlagString(flags, "limit")); if (hasFlag(flags, "dry-run")) { const cases = await loadEvalCases(inputPath, { limit }); console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`); return; } const { report, reportPath } = await runOnboardingEval({ enableLlmJudge: !hasFlag(flags, "skip-llm-judge"), inputPath, limit, outputPath: getFlagString(flags, "output"), setName, }); const publishedReport = hasFlag(flags, "no-publish-report") ? false : await publishEvalReportIfPossible(report); printRunSummary(report, reportPath); console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`); return; } if (command === "baseline") { const { updateEvalBaseline } = await import("./runner.js"); const reportPath = getFlagString(flags, "report") ?? positionals[0]; const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath); console.info( `[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`, ); return; } if (command === "report") { const { queryArchiveSliceReportRows } = await import("./archive.js"); const limit = parsePositiveInteger(getFlagString(flags, "worst")) ?? parsePositiveInteger(getFlagString(flags, "top")) ?? 20; const sort = hasFlag(flags, "worst") ? "score-asc" : "count-desc"; const mongo = await connectMongoForEval(); try { const rows = await queryArchiveSliceReportRows(mongo.connection, { limit, sinceMs: parseDurationMs(getFlagString(flags, "since")), sort, }); printSliceRows(rows); } finally { await mongo.close(); } return; } if (command === "promote") { const { appendCandidatesToJsonl, selectPromotionCandidates } = await import( "./promote.js" ); const mongo = await connectMongoForEval(); try { const candidates = await selectPromotionCandidates(mongo.connection, { limit: parsePositiveInteger(getFlagString(flags, "limit")), minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")), }); if (hasFlag(flags, "dry-run")) { for (const candidate of candidates) { const overall = candidate.llmJudge.skipped ? "n/a" : candidate.llmJudge.scores.overall; console.info( `[eval] candidate id=${candidate.caseId} reason=${candidate.reason} overall=${overall} trigger=${candidate.request.trigger}`, ); } console.info(`[eval] dryRunCandidates=${candidates.length}`); return; } const result = await appendCandidatesToJsonl( candidates, getFlagString(flags, "output"), ); console.info( `[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`, ); } finally { await mongo.close(); } return; } if (command === "review") { const { parseReviewReason, runReviewCli } = await import("./review/cli.js"); const garnet = await connectGarnetForEval(); try { const result = await runReviewCli(garnet.garnetClient, { exportLabeled: getFlagString(flags, "export-labeled"), limit: parsePositiveInteger(getFlagString(flags, "limit")), reason: parseReviewReason(getFlagString(flags, "reason")), resetPending: hasFlag(flags, "reset-pending"), }); console.info( `[eval] review completed labeled=${result.labeled} skipped=${result.skipped}`, ); } finally { await garnet.close(); } return; } if (command === "calibrate") { const { getEvalCalibrationMinCorr } = await import("../config.js"); const { computeCalibrationReport } = await import( "./review/calibration.js" ); const mongo = await connectMongoForEval(); const garnet = await connectGarnetForEval(); try { const report = await computeCalibrationReport( mongo.connection, garnet.garnetClient, { sinceMs: getFlagString(flags, "since") ? parseDurationMs(getFlagString(flags, "since")) : undefined, }, ); printCalibrationReport(report); if (report.pairs < 10) { console.warn( "[eval] calibration pairs<10; check gate is informational until at least 10 labels are available", ); } if ( hasFlag(flags, "check") && report.pairs >= 10 && report.pearsonOverall < getEvalCalibrationMinCorr() ) { process.exitCode = 1; } } finally { await garnet.close(); await mongo.close(); } return; } if (command === "train-prm") { const { getEvalPrmMinLabels, getEvalPrmWeightsPath, } = await import("../config.js"); const { getEvalArchiveCollection } = await import("./archive.js"); const { queryArchivedReviewLabels } = await import( "./review/label-archive.js" ); const { buildPrmTrainingSamples, savePrmModel, trainRidgePrmModel, } = await import("./judges/prm.js"); const mongo = await connectMongoForEval(); try { const labels = await queryArchivedReviewLabels(mongo.connection, { limit: parsePositiveInteger(getFlagString(flags, "label-limit")), }); const minLabels = parsePositiveInteger(getFlagString(flags, "min-labels")) ?? getEvalPrmMinLabels(); if (labels.length < minLabels && !hasFlag(flags, "force")) { throw new Error( `Not enough review labels for PRM training: ${labels.length}/${minLabels}. Use --force to override.`, ); } const archiveLimit = parsePositiveInteger(getFlagString(flags, "archive-limit")) ?? Math.max(labels.length, minLabels); const archives = await getEvalArchiveCollection(mongo.connection) .find({ streamId: { $in: labels.map((label) => label.evalStreamId), }, }) .sort({ evaluatedAt: -1 }) .limit(archiveLimit) .toArray(); const samples = buildPrmTrainingSamples(labels, archives); if (samples.length < minLabels && !hasFlag(flags, "force")) { throw new Error( `Not enough joined PRM samples: ${samples.length}/${minLabels}. Use --force to override.`, ); } const outputPath = getFlagString(flags, "output") ?? getEvalPrmWeightsPath(); if (!outputPath) { throw new Error( "Missing PRM output path. Set --output or EVAL_PRM_WEIGHTS_PATH.", ); } const model = trainRidgePrmModel(samples, { lambda: parsePositiveNumber(getFlagString(flags, "lambda")), }); savePrmModel(outputPath, model); console.info( `[eval] prm trained labels=${labels.length} archives=${archives.length} samples=${samples.length} output=${outputPath}`, ); } finally { await mongo.close(); } return; } if (command === "train-router") { const { getAgentRouterWeightsPath, getEvalPrmMinLabels, } = await import("../config.js"); const { getEvalArchiveCollection } = await import("./archive.js"); const { buildRouterTrainingSamples } = await import("./router-training.js"); const { saveLearnedRouterModel, trainLearnedRouterModel, } = await import("@skills/onboarding/runtime/router"); const mongo = await connectMongoForEval(); try { const archiveLimit = parsePositiveInteger(getFlagString(flags, "archive-limit")) ?? parsePositiveInteger(getFlagString(flags, "limit")) ?? 5_000; const minSamples = parsePositiveInteger(getFlagString(flags, "min-samples")) ?? getEvalPrmMinLabels(); const archives = await getEvalArchiveCollection(mongo.connection) .find({}) .sort({ evaluatedAt: -1 }) .limit(archiveLimit) .toArray(); const samples = buildRouterTrainingSamples(archives); if (samples.length < minSamples && !hasFlag(flags, "force")) { throw new Error( `Not enough router training samples: ${samples.length}/${minSamples}. Use --force to override.`, ); } const outputPath = getFlagString(flags, "output") ?? getAgentRouterWeightsPath(); if (!outputPath) { throw new Error( "Missing router output path. Set --output or AGENT_ROUTER_WEIGHTS_PATH.", ); } const model = trainLearnedRouterModel(samples, { epochs: parsePositiveInteger(getFlagString(flags, "epochs")), learningRate: parsePositiveNumber(getFlagString(flags, "learning-rate")), l2: parsePositiveNumber(getFlagString(flags, "l2")), }); saveLearnedRouterModel(outputPath, model); console.info( `[eval] router trained archives=${archives.length} samples=${samples.length} output=${outputPath}`, ); } finally { await mongo.close(); } return; } if (command === "propose") { const { createEvalLlmClient } = await import("./runner.js"); const { buildProposalInputFromArchive, proposeVariants, writePromptVariantProposal, } = await import("./loop/propose.js"); const mongo = await connectMongoForEval(); try { const input = await buildProposalInputFromArchive( mongo.connection, parseSlice(getFlagString(flags, "slice")), { force: hasFlag(flags, "force"), limit: parsePositiveInteger(getFlagString(flags, "limit")), }, ); const proposals = await proposeVariants(createEvalLlmClient(), input); for (const proposal of proposals.prompts) { const outputPath = await writePromptVariantProposal( proposal, getFlagString(flags, "output-dir"), ); console.info( `[eval] proposal variant=${proposal.variantId} output=${outputPath}`, ); } for (const tool of proposals.tools) { console.info( `[eval] toolProposal variant=${tool.variantId} tool=${tool.toolName} why=${tool.whyNeeded}`, ); } } finally { await mongo.close(); } return; } if (command === "tournament") { const { createEvalLlmClient } = await import("./runner.js"); const { loadTournamentCases, runTournament, saveTournamentResult, } = await import("./loop/tournament.js"); const variantIds = parseCsv( getFlagString(flags, "variants") ?? positionals[0], ); if (variantIds.length < 2) { throw new Error( 'eval:tournament requires --variants=control,', ); } const mongo = await connectMongoForEval(); try { const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches")); const cases = await loadTournamentCases(mongo.connection, { caseIds: parseCsv(getFlagString(flags, "case-ids")), limit: maxMatches, }); const result = await runTournament(createEvalLlmClient(), { cases, maxMatches, variantIds, }); await saveTournamentResult(mongo.connection, result); printTournamentResult(result); } finally { await mongo.close(); } return; } if (command === "tournament-results") { const { queryTournamentResults } = await import("./loop/tournament.js"); const mongo = await connectMongoForEval(); try { const results = await queryTournamentResults(mongo.connection, { limit: parsePositiveInteger(getFlagString(flags, "limit")), variantId: getFlagString(flags, "variant"), }); for (const result of results) { printTournamentResult(result); } } finally { await mongo.close(); } return; } throw new Error(`Unknown eval command: ${command}`); } catch (error) { logger.error( { err: error }, `[eval] "${command}" failed`, ); process.exitCode = 1; } } void main();