Files
lolympus-agent/src/eval/cli.ts
T

708 lines
22 KiB
TypeScript

import type { MongoConnectionLike } from "./archive.js";
import type { SliceKey, SliceReportRow } from "./aggregator.js";
import type { GarnetClient } from "@common/garnet";
function parsePositiveInteger(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
function parsePositiveNumber(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
}
function parseDurationMs(value: string | undefined): number {
const raw = value?.trim().toLowerCase() || "7d";
const match = raw.match(/^(\d+)(h|d)$/u);
if (!match) {
throw new Error("--since must use a duration like 24h, 7d, or 30d");
}
const amount = Number(match[1]);
const unit = match[2];
if (!Number.isInteger(amount) || amount <= 0) {
throw new Error("--since must be a positive duration");
}
return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000;
}
function parseCsv(value: string | undefined): string[] {
if (!value?.trim()) {
return [];
}
return value
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
function parseSlice(value: string | undefined): SliceKey {
if (!value?.trim()) {
throw new Error(
"--slice must use tier=0,intent=status-question,phase=photo-pending",
);
}
const fields: Record<string, string> = {};
for (const pair of value.split(",")) {
const [key, ...rest] = pair.split("=");
const normalizedKey = key?.trim();
if (normalizedKey) {
fields[normalizedKey] = rest.join("=").trim();
}
}
const tier = Number(fields["tier"]);
const intent = fields["intent"];
const phase = fields["phase"];
if (
(tier !== 0 && tier !== 1 && tier !== 2) ||
!intent?.trim() ||
!phase?.trim()
) {
throw new Error(
"--slice must include tier=0|1|2,intent=<kind>,phase=<kind>",
);
}
return {
tier,
intent,
phase,
};
}
interface ParsedArgs {
command: string;
flags: Map<string, string | true>;
positionals: string[];
}
function parseArgs(argv: string[]): ParsedArgs {
const [command = "run", ...rest] = argv;
const flags = new Map<string, string | true>();
const positionals: string[] = [];
for (let index = 0; index < rest.length; index++) {
const token = rest[index];
if (!token) {
continue;
}
if (!token.startsWith("--")) {
positionals.push(token);
continue;
}
const withoutPrefix = token.slice(2);
const equalsIndex = withoutPrefix.indexOf("=");
if (equalsIndex >= 0) {
const key = withoutPrefix.slice(0, equalsIndex);
const value = withoutPrefix.slice(equalsIndex + 1);
flags.set(key, value || true);
continue;
}
const next = rest[index + 1];
if (next && !next.startsWith("--")) {
flags.set(withoutPrefix, next);
index += 1;
continue;
}
flags.set(withoutPrefix, true);
}
return {
command,
flags,
positionals,
};
}
function getFlagString(
flags: Map<string, string | true>,
key: string,
): string | undefined {
const value = flags.get(key);
return typeof value === "string" ? value : undefined;
}
function hasFlag(
flags: Map<string, string | true>,
key: string,
): boolean {
return flags.has(key);
}
function printRunSummary(report: {
generatedAt: string;
summary: {
cases: number;
passRate: number;
ruleAverageScore: number;
averageLatencyMs: number;
llmAverageOverall: number | null;
llmJudgedCases: number;
};
baselineDelta?: {
passRate: number | null;
ruleAverageScore: number | null;
llmAverageOverall: number | null;
averageLatencyMs: number | null;
} | undefined;
}, reportPath: string): void {
const llmLine =
report.summary.llmAverageOverall === null
? "n/a"
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
console.info(`[eval] generatedAt=${report.generatedAt}`);
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
console.info(
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
);
console.info(`[eval] llmAverageOverall=${llmLine}`);
if (report.baselineDelta) {
console.info(
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
);
}
console.info(`[eval] report=${reportPath}`);
}
function formatNullable(value: number | null): string {
return value === null ? "n/a" : String(value);
}
function printSliceRows(rows: SliceReportRow[]): void {
console.info(
[
"tier",
"intent",
"phase",
"count",
"llmOverallAvg",
"validatorRejectRate",
"p50LatencyMs",
"p95LatencyMs",
"updatedAt",
].join("\t"),
);
for (const row of rows) {
console.info(
[
row.slice.tier,
row.slice.intent,
row.slice.phase,
row.count,
formatNullable(row.llmOverallAvg),
row.validatorRejectRate,
formatNullable(row.p50LatencyMs),
formatNullable(row.p95LatencyMs),
row.updatedAt,
].join("\t"),
);
}
}
function printCalibrationReport(report: {
generatedAt: string;
meanGapOverall: number;
pairs: number;
pearsonByDim: {
accuracy: number;
concision: number;
friendliness: number;
scope: number;
};
pearsonOverall: number;
worstDisagreements: Array<{
evalStreamId: string;
humanScore: number;
llmOverall: number;
}>;
}): void {
console.info(`[eval] generatedAt=${report.generatedAt}`);
console.info(`[eval] pairs=${report.pairs}`);
console.info(`[eval] pearsonOverall=${report.pearsonOverall}`);
console.info(`[eval] meanGapOverall=${report.meanGapOverall}`);
console.info(
`[eval] pearsonByDim friendliness=${report.pearsonByDim.friendliness} accuracy=${report.pearsonByDim.accuracy} concision=${report.pearsonByDim.concision} scope=${report.pearsonByDim.scope}`,
);
for (const item of report.worstDisagreements) {
console.info(
`[eval] disagreement evalStreamId=${item.evalStreamId} human=${item.humanScore} llm=${item.llmOverall}`,
);
}
}
function printTournamentResult(result: {
generatedAt: string;
matches: Array<{ judgeVerdict: string; variantA: string; variantB: string }>;
variants: Array<{ elo: number; id: string; matches: number; wins: number }>;
}): void {
console.info(`[eval] generatedAt=${result.generatedAt}`);
console.info(`[eval] matches=${result.matches.length}`);
for (const variant of result.variants) {
console.info(
`[eval] variant id=${variant.id} elo=${variant.elo} matches=${variant.matches} wins=${variant.wins}`,
);
}
}
async function connectMongoForEval(): Promise<{
close: () => Promise<void>;
connection: MongoConnectionLike;
}> {
const { initMongoDB, closeMongoDB } = await import("@common/mongodb");
const { getSecret } = await import("@common/secret-manager");
const connectionString =
getSecret("MONGODB_CONNECTION_STRING") ??
process.env["MONGODB_CONNECTION_STRING"];
if (!connectionString) {
throw new Error(
"Missing MONGODB_CONNECTION_STRING for eval command. Set it in the environment or secrets.",
);
}
return {
connection: await initMongoDB(connectionString),
close: closeMongoDB,
};
}
async function connectGarnetForEval(): Promise<{
close: () => Promise<void>;
garnetClient: GarnetClient;
}> {
const { GarnetClient } = await import("@common/garnet");
const garnetClient = new GarnetClient();
await garnetClient.connect();
return {
garnetClient,
close: () => garnetClient.close(),
};
}
async function main(): Promise<void> {
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
loadEnvironmentFromNodeEnv();
const { getLogger, initLogger } = await import("@obs/logger");
initLogger();
const logger = getLogger();
const parsed = parseArgs(process.argv.slice(2));
const { command, flags, positionals } = parsed;
try {
if (command === "capture") {
const { captureGoldenSetScaffold } = await import("./capture.js");
const outputPath = await captureGoldenSetScaffold({
limit:
parsePositiveInteger(getFlagString(flags, "limit")) ??
parsePositiveInteger(positionals[0]),
outputPath: getFlagString(flags, "output") ?? positionals[1],
});
logger.info({ outputPath }, '[eval] "capture" completed');
return;
}
if (command === "run") {
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
"./runner.js"
);
const { publishEvalReportIfPossible } = await import(
"./runner.js"
);
const setName = getFlagString(flags, "set") ?? "golden";
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
if (hasFlag(flags, "dry-run")) {
const cases = await loadEvalCases(inputPath, { limit });
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
return;
}
const { report, reportPath } = await runOnboardingEval({
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
inputPath,
limit,
outputPath: getFlagString(flags, "output"),
setName,
});
const publishedReport = hasFlag(flags, "no-publish-report")
? false
: await publishEvalReportIfPossible(report);
printRunSummary(report, reportPath);
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
return;
}
if (command === "baseline") {
const { updateEvalBaseline } = await import("./runner.js");
const reportPath = getFlagString(flags, "report") ?? positionals[0];
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
console.info(
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
);
return;
}
if (command === "report") {
const { queryArchiveSliceReportRows } = await import("./archive.js");
const limit =
parsePositiveInteger(getFlagString(flags, "worst")) ??
parsePositiveInteger(getFlagString(flags, "top")) ??
20;
const sort = hasFlag(flags, "worst") ? "score-asc" : "count-desc";
const mongo = await connectMongoForEval();
try {
const rows = await queryArchiveSliceReportRows(mongo.connection, {
limit,
sinceMs: parseDurationMs(getFlagString(flags, "since")),
sort,
});
printSliceRows(rows);
} finally {
await mongo.close();
}
return;
}
if (command === "promote") {
const { appendCandidatesToJsonl, selectPromotionCandidates } = await import(
"./promote.js"
);
const mongo = await connectMongoForEval();
try {
const candidates = await selectPromotionCandidates(mongo.connection, {
limit: parsePositiveInteger(getFlagString(flags, "limit")),
minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")),
});
if (hasFlag(flags, "dry-run")) {
for (const candidate of candidates) {
const overall = candidate.llmJudge.skipped
? "n/a"
: candidate.llmJudge.scores.overall;
console.info(
`[eval] candidate id=${candidate.caseId} reason=${candidate.reason} overall=${overall} trigger=${candidate.request.trigger}`,
);
}
console.info(`[eval] dryRunCandidates=${candidates.length}`);
return;
}
const result = await appendCandidatesToJsonl(
candidates,
getFlagString(flags, "output"),
);
console.info(
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
);
} finally {
await mongo.close();
}
return;
}
if (command === "review") {
const { parseReviewReason, runReviewCli } = await import("./review/cli.js");
const garnet = await connectGarnetForEval();
try {
const result = await runReviewCli(garnet.garnetClient, {
exportLabeled: getFlagString(flags, "export-labeled"),
limit: parsePositiveInteger(getFlagString(flags, "limit")),
reason: parseReviewReason(getFlagString(flags, "reason")),
resetPending: hasFlag(flags, "reset-pending"),
});
console.info(
`[eval] review completed labeled=${result.labeled} skipped=${result.skipped}`,
);
} finally {
await garnet.close();
}
return;
}
if (command === "calibrate") {
const { getEvalCalibrationMinCorr } = await import("../config.js");
const { computeCalibrationReport } = await import(
"./review/calibration.js"
);
const mongo = await connectMongoForEval();
const garnet = await connectGarnetForEval();
try {
const report = await computeCalibrationReport(
mongo.connection,
garnet.garnetClient,
{
sinceMs: getFlagString(flags, "since")
? parseDurationMs(getFlagString(flags, "since"))
: undefined,
},
);
printCalibrationReport(report);
if (report.pairs < 10) {
console.warn(
"[eval] calibration pairs<10; check gate is informational until at least 10 labels are available",
);
}
if (
hasFlag(flags, "check") &&
report.pairs >= 10 &&
report.pearsonOverall < getEvalCalibrationMinCorr()
) {
process.exitCode = 1;
}
} finally {
await garnet.close();
await mongo.close();
}
return;
}
if (command === "train-prm") {
const {
getEvalPrmMinLabels,
getEvalPrmWeightsPath,
} = await import("../config.js");
const { getEvalArchiveCollection } = await import("./archive.js");
const { queryArchivedReviewLabels } = await import(
"./review/label-archive.js"
);
const {
buildPrmTrainingSamples,
savePrmModel,
trainRidgePrmModel,
} = await import("./judges/prm.js");
const mongo = await connectMongoForEval();
try {
const labels = await queryArchivedReviewLabels(mongo.connection, {
limit: parsePositiveInteger(getFlagString(flags, "label-limit")),
});
const minLabels =
parsePositiveInteger(getFlagString(flags, "min-labels")) ??
getEvalPrmMinLabels();
if (labels.length < minLabels && !hasFlag(flags, "force")) {
throw new Error(
`Not enough review labels for PRM training: ${labels.length}/${minLabels}. Use --force to override.`,
);
}
const archiveLimit =
parsePositiveInteger(getFlagString(flags, "archive-limit")) ??
Math.max(labels.length, minLabels);
const archives = await getEvalArchiveCollection(mongo.connection)
.find({
streamId: {
$in: labels.map((label) => label.evalStreamId),
},
})
.sort({ evaluatedAt: -1 })
.limit(archiveLimit)
.toArray();
const samples = buildPrmTrainingSamples(labels, archives);
if (samples.length < minLabels && !hasFlag(flags, "force")) {
throw new Error(
`Not enough joined PRM samples: ${samples.length}/${minLabels}. Use --force to override.`,
);
}
const outputPath =
getFlagString(flags, "output") ?? getEvalPrmWeightsPath();
if (!outputPath) {
throw new Error(
"Missing PRM output path. Set --output or EVAL_PRM_WEIGHTS_PATH.",
);
}
const model = trainRidgePrmModel(samples, {
lambda: parsePositiveNumber(getFlagString(flags, "lambda")),
});
savePrmModel(outputPath, model);
console.info(
`[eval] prm trained labels=${labels.length} archives=${archives.length} samples=${samples.length} output=${outputPath}`,
);
} finally {
await mongo.close();
}
return;
}
if (command === "train-router") {
const {
getAgentRouterWeightsPath,
getEvalPrmMinLabels,
} = await import("../config.js");
const { getEvalArchiveCollection } = await import("./archive.js");
const { buildRouterTrainingSamples } = await import("./router-training.js");
const {
saveLearnedRouterModel,
trainLearnedRouterModel,
} = await import("@skills/onboarding/runtime/router");
const mongo = await connectMongoForEval();
try {
const archiveLimit =
parsePositiveInteger(getFlagString(flags, "archive-limit")) ??
parsePositiveInteger(getFlagString(flags, "limit")) ??
5_000;
const minSamples =
parsePositiveInteger(getFlagString(flags, "min-samples")) ??
getEvalPrmMinLabels();
const archives = await getEvalArchiveCollection(mongo.connection)
.find({})
.sort({ evaluatedAt: -1 })
.limit(archiveLimit)
.toArray();
const samples = buildRouterTrainingSamples(archives);
if (samples.length < minSamples && !hasFlag(flags, "force")) {
throw new Error(
`Not enough router training samples: ${samples.length}/${minSamples}. Use --force to override.`,
);
}
const outputPath =
getFlagString(flags, "output") ?? getAgentRouterWeightsPath();
if (!outputPath) {
throw new Error(
"Missing router output path. Set --output or AGENT_ROUTER_WEIGHTS_PATH.",
);
}
const model = trainLearnedRouterModel(samples, {
epochs: parsePositiveInteger(getFlagString(flags, "epochs")),
learningRate: parsePositiveNumber(getFlagString(flags, "learning-rate")),
l2: parsePositiveNumber(getFlagString(flags, "l2")),
});
saveLearnedRouterModel(outputPath, model);
console.info(
`[eval] router trained archives=${archives.length} samples=${samples.length} output=${outputPath}`,
);
} finally {
await mongo.close();
}
return;
}
if (command === "propose") {
const { createEvalLlmClient } = await import("./runner.js");
const {
buildProposalInputFromArchive,
proposeVariants,
writePromptVariantProposal,
} = await import("./loop/propose.js");
const mongo = await connectMongoForEval();
try {
const input = await buildProposalInputFromArchive(
mongo.connection,
parseSlice(getFlagString(flags, "slice")),
{
force: hasFlag(flags, "force"),
limit: parsePositiveInteger(getFlagString(flags, "limit")),
},
);
const proposals = await proposeVariants(createEvalLlmClient(), input);
for (const proposal of proposals.prompts) {
const outputPath = await writePromptVariantProposal(
proposal,
getFlagString(flags, "output-dir"),
);
console.info(
`[eval] proposal variant=${proposal.variantId} output=${outputPath}`,
);
}
for (const tool of proposals.tools) {
console.info(
`[eval] toolProposal variant=${tool.variantId} tool=${tool.toolName} why=${tool.whyNeeded}`,
);
}
} finally {
await mongo.close();
}
return;
}
if (command === "tournament") {
const { createEvalLlmClient } = await import("./runner.js");
const {
loadTournamentCases,
runTournament,
saveTournamentResult,
} = await import("./loop/tournament.js");
const variantIds = parseCsv(
getFlagString(flags, "variants") ?? positionals[0],
);
if (variantIds.length < 2) {
throw new Error(
'eval:tournament requires --variants=control,<variantId>',
);
}
const mongo = await connectMongoForEval();
try {
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
const cases = await loadTournamentCases(mongo.connection, {
caseIds: parseCsv(getFlagString(flags, "case-ids")),
limit: maxMatches,
});
const result = await runTournament(createEvalLlmClient(), {
cases,
maxMatches,
variantIds,
});
await saveTournamentResult(mongo.connection, result);
printTournamentResult(result);
} finally {
await mongo.close();
}
return;
}
if (command === "tournament-results") {
const { queryTournamentResults } = await import("./loop/tournament.js");
const mongo = await connectMongoForEval();
try {
const results = await queryTournamentResults(mongo.connection, {
limit: parsePositiveInteger(getFlagString(flags, "limit")),
variantId: getFlagString(flags, "variant"),
});
for (const result of results) {
printTournamentResult(result);
}
} finally {
await mongo.close();
}
return;
}
throw new Error(`Unknown eval command: ${command}`);
} catch (error) {
logger.error(
{ err: error },
`[eval] "${command}" failed`,
);
process.exitCode = 1;
}
}
void main();