708 lines
22 KiB
TypeScript
708 lines
22 KiB
TypeScript
import type { MongoConnectionLike } from "./archive.js";
|
|
import type { SliceKey, SliceReportRow } from "./aggregator.js";
|
|
import type { GarnetClient } from "@common/garnet";
|
|
|
|
function parsePositiveInteger(value: string | undefined): number | undefined {
|
|
if (!value?.trim()) {
|
|
return undefined;
|
|
}
|
|
|
|
const parsed = Number(value);
|
|
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
|
}
|
|
|
|
function parsePositiveNumber(value: string | undefined): number | undefined {
|
|
if (!value?.trim()) {
|
|
return undefined;
|
|
}
|
|
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
}
|
|
|
|
function parseDurationMs(value: string | undefined): number {
|
|
const raw = value?.trim().toLowerCase() || "7d";
|
|
const match = raw.match(/^(\d+)(h|d)$/u);
|
|
if (!match) {
|
|
throw new Error("--since must use a duration like 24h, 7d, or 30d");
|
|
}
|
|
|
|
const amount = Number(match[1]);
|
|
const unit = match[2];
|
|
if (!Number.isInteger(amount) || amount <= 0) {
|
|
throw new Error("--since must be a positive duration");
|
|
}
|
|
|
|
return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000;
|
|
}
|
|
|
|
function parseCsv(value: string | undefined): string[] {
|
|
if (!value?.trim()) {
|
|
return [];
|
|
}
|
|
|
|
return value
|
|
.split(",")
|
|
.map((item) => item.trim())
|
|
.filter((item) => item.length > 0);
|
|
}
|
|
|
|
function parseSlice(value: string | undefined): SliceKey {
|
|
if (!value?.trim()) {
|
|
throw new Error(
|
|
"--slice must use tier=0,intent=status-question,phase=photo-pending",
|
|
);
|
|
}
|
|
|
|
const fields: Record<string, string> = {};
|
|
for (const pair of value.split(",")) {
|
|
const [key, ...rest] = pair.split("=");
|
|
const normalizedKey = key?.trim();
|
|
if (normalizedKey) {
|
|
fields[normalizedKey] = rest.join("=").trim();
|
|
}
|
|
}
|
|
const tier = Number(fields["tier"]);
|
|
const intent = fields["intent"];
|
|
const phase = fields["phase"];
|
|
|
|
if (
|
|
(tier !== 0 && tier !== 1 && tier !== 2) ||
|
|
!intent?.trim() ||
|
|
!phase?.trim()
|
|
) {
|
|
throw new Error(
|
|
"--slice must include tier=0|1|2,intent=<kind>,phase=<kind>",
|
|
);
|
|
}
|
|
|
|
return {
|
|
tier,
|
|
intent,
|
|
phase,
|
|
};
|
|
}
|
|
|
|
interface ParsedArgs {
|
|
command: string;
|
|
flags: Map<string, string | true>;
|
|
positionals: string[];
|
|
}
|
|
|
|
function parseArgs(argv: string[]): ParsedArgs {
|
|
const [command = "run", ...rest] = argv;
|
|
const flags = new Map<string, string | true>();
|
|
const positionals: string[] = [];
|
|
|
|
for (let index = 0; index < rest.length; index++) {
|
|
const token = rest[index];
|
|
if (!token) {
|
|
continue;
|
|
}
|
|
|
|
if (!token.startsWith("--")) {
|
|
positionals.push(token);
|
|
continue;
|
|
}
|
|
|
|
const withoutPrefix = token.slice(2);
|
|
const equalsIndex = withoutPrefix.indexOf("=");
|
|
if (equalsIndex >= 0) {
|
|
const key = withoutPrefix.slice(0, equalsIndex);
|
|
const value = withoutPrefix.slice(equalsIndex + 1);
|
|
flags.set(key, value || true);
|
|
continue;
|
|
}
|
|
|
|
const next = rest[index + 1];
|
|
if (next && !next.startsWith("--")) {
|
|
flags.set(withoutPrefix, next);
|
|
index += 1;
|
|
continue;
|
|
}
|
|
|
|
flags.set(withoutPrefix, true);
|
|
}
|
|
|
|
return {
|
|
command,
|
|
flags,
|
|
positionals,
|
|
};
|
|
}
|
|
|
|
function getFlagString(
|
|
flags: Map<string, string | true>,
|
|
key: string,
|
|
): string | undefined {
|
|
const value = flags.get(key);
|
|
return typeof value === "string" ? value : undefined;
|
|
}
|
|
|
|
function hasFlag(
|
|
flags: Map<string, string | true>,
|
|
key: string,
|
|
): boolean {
|
|
return flags.has(key);
|
|
}
|
|
|
|
function printRunSummary(report: {
|
|
generatedAt: string;
|
|
summary: {
|
|
cases: number;
|
|
passRate: number;
|
|
ruleAverageScore: number;
|
|
averageLatencyMs: number;
|
|
llmAverageOverall: number | null;
|
|
llmJudgedCases: number;
|
|
};
|
|
baselineDelta?: {
|
|
passRate: number | null;
|
|
ruleAverageScore: number | null;
|
|
llmAverageOverall: number | null;
|
|
averageLatencyMs: number | null;
|
|
} | undefined;
|
|
}, reportPath: string): void {
|
|
const llmLine =
|
|
report.summary.llmAverageOverall === null
|
|
? "n/a"
|
|
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
|
|
|
|
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
|
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
|
|
console.info(
|
|
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
|
|
);
|
|
console.info(`[eval] llmAverageOverall=${llmLine}`);
|
|
|
|
if (report.baselineDelta) {
|
|
console.info(
|
|
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
|
|
);
|
|
}
|
|
|
|
console.info(`[eval] report=${reportPath}`);
|
|
}
|
|
|
|
function formatNullable(value: number | null): string {
|
|
return value === null ? "n/a" : String(value);
|
|
}
|
|
|
|
function printSliceRows(rows: SliceReportRow[]): void {
|
|
console.info(
|
|
[
|
|
"tier",
|
|
"intent",
|
|
"phase",
|
|
"count",
|
|
"llmOverallAvg",
|
|
"validatorRejectRate",
|
|
"p50LatencyMs",
|
|
"p95LatencyMs",
|
|
"updatedAt",
|
|
].join("\t"),
|
|
);
|
|
|
|
for (const row of rows) {
|
|
console.info(
|
|
[
|
|
row.slice.tier,
|
|
row.slice.intent,
|
|
row.slice.phase,
|
|
row.count,
|
|
formatNullable(row.llmOverallAvg),
|
|
row.validatorRejectRate,
|
|
formatNullable(row.p50LatencyMs),
|
|
formatNullable(row.p95LatencyMs),
|
|
row.updatedAt,
|
|
].join("\t"),
|
|
);
|
|
}
|
|
}
|
|
|
|
function printCalibrationReport(report: {
|
|
generatedAt: string;
|
|
meanGapOverall: number;
|
|
pairs: number;
|
|
pearsonByDim: {
|
|
accuracy: number;
|
|
concision: number;
|
|
friendliness: number;
|
|
scope: number;
|
|
};
|
|
pearsonOverall: number;
|
|
worstDisagreements: Array<{
|
|
evalStreamId: string;
|
|
humanScore: number;
|
|
llmOverall: number;
|
|
}>;
|
|
}): void {
|
|
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
|
console.info(`[eval] pairs=${report.pairs}`);
|
|
console.info(`[eval] pearsonOverall=${report.pearsonOverall}`);
|
|
console.info(`[eval] meanGapOverall=${report.meanGapOverall}`);
|
|
console.info(
|
|
`[eval] pearsonByDim friendliness=${report.pearsonByDim.friendliness} accuracy=${report.pearsonByDim.accuracy} concision=${report.pearsonByDim.concision} scope=${report.pearsonByDim.scope}`,
|
|
);
|
|
for (const item of report.worstDisagreements) {
|
|
console.info(
|
|
`[eval] disagreement evalStreamId=${item.evalStreamId} human=${item.humanScore} llm=${item.llmOverall}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function printTournamentResult(result: {
|
|
generatedAt: string;
|
|
matches: Array<{ judgeVerdict: string; variantA: string; variantB: string }>;
|
|
variants: Array<{ elo: number; id: string; matches: number; wins: number }>;
|
|
}): void {
|
|
console.info(`[eval] generatedAt=${result.generatedAt}`);
|
|
console.info(`[eval] matches=${result.matches.length}`);
|
|
for (const variant of result.variants) {
|
|
console.info(
|
|
`[eval] variant id=${variant.id} elo=${variant.elo} matches=${variant.matches} wins=${variant.wins}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
async function connectMongoForEval(): Promise<{
|
|
close: () => Promise<void>;
|
|
connection: MongoConnectionLike;
|
|
}> {
|
|
const { initMongoDB, closeMongoDB } = await import("@common/mongodb");
|
|
const { getSecret } = await import("@common/secret-manager");
|
|
const connectionString =
|
|
getSecret("MONGODB_CONNECTION_STRING") ??
|
|
process.env["MONGODB_CONNECTION_STRING"];
|
|
|
|
if (!connectionString) {
|
|
throw new Error(
|
|
"Missing MONGODB_CONNECTION_STRING for eval command. Set it in the environment or secrets.",
|
|
);
|
|
}
|
|
|
|
return {
|
|
connection: await initMongoDB(connectionString),
|
|
close: closeMongoDB,
|
|
};
|
|
}
|
|
|
|
async function connectGarnetForEval(): Promise<{
|
|
close: () => Promise<void>;
|
|
garnetClient: GarnetClient;
|
|
}> {
|
|
const { GarnetClient } = await import("@common/garnet");
|
|
const garnetClient = new GarnetClient();
|
|
await garnetClient.connect();
|
|
return {
|
|
garnetClient,
|
|
close: () => garnetClient.close(),
|
|
};
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
|
|
loadEnvironmentFromNodeEnv();
|
|
|
|
const { getLogger, initLogger } = await import("@obs/logger");
|
|
initLogger();
|
|
const logger = getLogger();
|
|
|
|
const parsed = parseArgs(process.argv.slice(2));
|
|
const { command, flags, positionals } = parsed;
|
|
|
|
try {
|
|
if (command === "capture") {
|
|
const { captureGoldenSetScaffold } = await import("./capture.js");
|
|
const outputPath = await captureGoldenSetScaffold({
|
|
limit:
|
|
parsePositiveInteger(getFlagString(flags, "limit")) ??
|
|
parsePositiveInteger(positionals[0]),
|
|
outputPath: getFlagString(flags, "output") ?? positionals[1],
|
|
});
|
|
logger.info({ outputPath }, '[eval] "capture" completed');
|
|
return;
|
|
}
|
|
|
|
if (command === "run") {
|
|
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
|
|
"./runner.js"
|
|
);
|
|
const { publishEvalReportIfPossible } = await import(
|
|
"./runner.js"
|
|
);
|
|
const setName = getFlagString(flags, "set") ?? "golden";
|
|
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
|
|
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
|
|
|
|
if (hasFlag(flags, "dry-run")) {
|
|
const cases = await loadEvalCases(inputPath, { limit });
|
|
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
|
|
return;
|
|
}
|
|
|
|
const { report, reportPath } = await runOnboardingEval({
|
|
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
|
|
inputPath,
|
|
limit,
|
|
outputPath: getFlagString(flags, "output"),
|
|
setName,
|
|
});
|
|
const publishedReport = hasFlag(flags, "no-publish-report")
|
|
? false
|
|
: await publishEvalReportIfPossible(report);
|
|
printRunSummary(report, reportPath);
|
|
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
|
|
return;
|
|
}
|
|
|
|
if (command === "baseline") {
|
|
const { updateEvalBaseline } = await import("./runner.js");
|
|
const reportPath = getFlagString(flags, "report") ?? positionals[0];
|
|
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
|
|
console.info(
|
|
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
if (command === "report") {
|
|
const { queryArchiveSliceReportRows } = await import("./archive.js");
|
|
const limit =
|
|
parsePositiveInteger(getFlagString(flags, "worst")) ??
|
|
parsePositiveInteger(getFlagString(flags, "top")) ??
|
|
20;
|
|
const sort = hasFlag(flags, "worst") ? "score-asc" : "count-desc";
|
|
const mongo = await connectMongoForEval();
|
|
try {
|
|
const rows = await queryArchiveSliceReportRows(mongo.connection, {
|
|
limit,
|
|
sinceMs: parseDurationMs(getFlagString(flags, "since")),
|
|
sort,
|
|
});
|
|
printSliceRows(rows);
|
|
} finally {
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "promote") {
|
|
const { appendCandidatesToJsonl, selectPromotionCandidates } = await import(
|
|
"./promote.js"
|
|
);
|
|
const mongo = await connectMongoForEval();
|
|
try {
|
|
const candidates = await selectPromotionCandidates(mongo.connection, {
|
|
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
|
minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")),
|
|
});
|
|
|
|
if (hasFlag(flags, "dry-run")) {
|
|
for (const candidate of candidates) {
|
|
const overall = candidate.llmJudge.skipped
|
|
? "n/a"
|
|
: candidate.llmJudge.scores.overall;
|
|
console.info(
|
|
`[eval] candidate id=${candidate.caseId} reason=${candidate.reason} overall=${overall} trigger=${candidate.request.trigger}`,
|
|
);
|
|
}
|
|
console.info(`[eval] dryRunCandidates=${candidates.length}`);
|
|
return;
|
|
}
|
|
|
|
const result = await appendCandidatesToJsonl(
|
|
candidates,
|
|
getFlagString(flags, "output"),
|
|
);
|
|
console.info(
|
|
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
|
|
);
|
|
} finally {
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "review") {
|
|
const { parseReviewReason, runReviewCli } = await import("./review/cli.js");
|
|
const garnet = await connectGarnetForEval();
|
|
try {
|
|
const result = await runReviewCli(garnet.garnetClient, {
|
|
exportLabeled: getFlagString(flags, "export-labeled"),
|
|
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
|
reason: parseReviewReason(getFlagString(flags, "reason")),
|
|
resetPending: hasFlag(flags, "reset-pending"),
|
|
});
|
|
console.info(
|
|
`[eval] review completed labeled=${result.labeled} skipped=${result.skipped}`,
|
|
);
|
|
} finally {
|
|
await garnet.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "calibrate") {
|
|
const { getEvalCalibrationMinCorr } = await import("../config.js");
|
|
const { computeCalibrationReport } = await import(
|
|
"./review/calibration.js"
|
|
);
|
|
const mongo = await connectMongoForEval();
|
|
const garnet = await connectGarnetForEval();
|
|
try {
|
|
const report = await computeCalibrationReport(
|
|
mongo.connection,
|
|
garnet.garnetClient,
|
|
{
|
|
sinceMs: getFlagString(flags, "since")
|
|
? parseDurationMs(getFlagString(flags, "since"))
|
|
: undefined,
|
|
},
|
|
);
|
|
printCalibrationReport(report);
|
|
if (report.pairs < 10) {
|
|
console.warn(
|
|
"[eval] calibration pairs<10; check gate is informational until at least 10 labels are available",
|
|
);
|
|
}
|
|
if (
|
|
hasFlag(flags, "check") &&
|
|
report.pairs >= 10 &&
|
|
report.pearsonOverall < getEvalCalibrationMinCorr()
|
|
) {
|
|
process.exitCode = 1;
|
|
}
|
|
} finally {
|
|
await garnet.close();
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "train-prm") {
|
|
const {
|
|
getEvalPrmMinLabels,
|
|
getEvalPrmWeightsPath,
|
|
} = await import("../config.js");
|
|
const { getEvalArchiveCollection } = await import("./archive.js");
|
|
const { queryArchivedReviewLabels } = await import(
|
|
"./review/label-archive.js"
|
|
);
|
|
const {
|
|
buildPrmTrainingSamples,
|
|
savePrmModel,
|
|
trainRidgePrmModel,
|
|
} = await import("./judges/prm.js");
|
|
const mongo = await connectMongoForEval();
|
|
try {
|
|
const labels = await queryArchivedReviewLabels(mongo.connection, {
|
|
limit: parsePositiveInteger(getFlagString(flags, "label-limit")),
|
|
});
|
|
const minLabels =
|
|
parsePositiveInteger(getFlagString(flags, "min-labels")) ??
|
|
getEvalPrmMinLabels();
|
|
if (labels.length < minLabels && !hasFlag(flags, "force")) {
|
|
throw new Error(
|
|
`Not enough review labels for PRM training: ${labels.length}/${minLabels}. Use --force to override.`,
|
|
);
|
|
}
|
|
|
|
const archiveLimit =
|
|
parsePositiveInteger(getFlagString(flags, "archive-limit")) ??
|
|
Math.max(labels.length, minLabels);
|
|
const archives = await getEvalArchiveCollection(mongo.connection)
|
|
.find({
|
|
streamId: {
|
|
$in: labels.map((label) => label.evalStreamId),
|
|
},
|
|
})
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(archiveLimit)
|
|
.toArray();
|
|
const samples = buildPrmTrainingSamples(labels, archives);
|
|
if (samples.length < minLabels && !hasFlag(flags, "force")) {
|
|
throw new Error(
|
|
`Not enough joined PRM samples: ${samples.length}/${minLabels}. Use --force to override.`,
|
|
);
|
|
}
|
|
|
|
const outputPath =
|
|
getFlagString(flags, "output") ?? getEvalPrmWeightsPath();
|
|
if (!outputPath) {
|
|
throw new Error(
|
|
"Missing PRM output path. Set --output or EVAL_PRM_WEIGHTS_PATH.",
|
|
);
|
|
}
|
|
|
|
const model = trainRidgePrmModel(samples, {
|
|
lambda: parsePositiveNumber(getFlagString(flags, "lambda")),
|
|
});
|
|
savePrmModel(outputPath, model);
|
|
console.info(
|
|
`[eval] prm trained labels=${labels.length} archives=${archives.length} samples=${samples.length} output=${outputPath}`,
|
|
);
|
|
} finally {
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "train-router") {
|
|
const {
|
|
getAgentRouterWeightsPath,
|
|
getEvalPrmMinLabels,
|
|
} = await import("../config.js");
|
|
const { getEvalArchiveCollection } = await import("./archive.js");
|
|
const { buildRouterTrainingSamples } = await import("./router-training.js");
|
|
const {
|
|
saveLearnedRouterModel,
|
|
trainLearnedRouterModel,
|
|
} = await import("@skills/onboarding/runtime/router");
|
|
const mongo = await connectMongoForEval();
|
|
try {
|
|
const archiveLimit =
|
|
parsePositiveInteger(getFlagString(flags, "archive-limit")) ??
|
|
parsePositiveInteger(getFlagString(flags, "limit")) ??
|
|
5_000;
|
|
const minSamples =
|
|
parsePositiveInteger(getFlagString(flags, "min-samples")) ??
|
|
getEvalPrmMinLabels();
|
|
const archives = await getEvalArchiveCollection(mongo.connection)
|
|
.find({})
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(archiveLimit)
|
|
.toArray();
|
|
const samples = buildRouterTrainingSamples(archives);
|
|
if (samples.length < minSamples && !hasFlag(flags, "force")) {
|
|
throw new Error(
|
|
`Not enough router training samples: ${samples.length}/${minSamples}. Use --force to override.`,
|
|
);
|
|
}
|
|
|
|
const outputPath =
|
|
getFlagString(flags, "output") ?? getAgentRouterWeightsPath();
|
|
if (!outputPath) {
|
|
throw new Error(
|
|
"Missing router output path. Set --output or AGENT_ROUTER_WEIGHTS_PATH.",
|
|
);
|
|
}
|
|
|
|
const model = trainLearnedRouterModel(samples, {
|
|
epochs: parsePositiveInteger(getFlagString(flags, "epochs")),
|
|
learningRate: parsePositiveNumber(getFlagString(flags, "learning-rate")),
|
|
l2: parsePositiveNumber(getFlagString(flags, "l2")),
|
|
});
|
|
saveLearnedRouterModel(outputPath, model);
|
|
console.info(
|
|
`[eval] router trained archives=${archives.length} samples=${samples.length} output=${outputPath}`,
|
|
);
|
|
} finally {
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "propose") {
|
|
const { createEvalLlmClient } = await import("./runner.js");
|
|
const {
|
|
buildProposalInputFromArchive,
|
|
proposeVariants,
|
|
writePromptVariantProposal,
|
|
} = await import("./loop/propose.js");
|
|
const mongo = await connectMongoForEval();
|
|
try {
|
|
const input = await buildProposalInputFromArchive(
|
|
mongo.connection,
|
|
parseSlice(getFlagString(flags, "slice")),
|
|
{
|
|
force: hasFlag(flags, "force"),
|
|
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
|
},
|
|
);
|
|
const proposals = await proposeVariants(createEvalLlmClient(), input);
|
|
for (const proposal of proposals.prompts) {
|
|
const outputPath = await writePromptVariantProposal(
|
|
proposal,
|
|
getFlagString(flags, "output-dir"),
|
|
);
|
|
console.info(
|
|
`[eval] proposal variant=${proposal.variantId} output=${outputPath}`,
|
|
);
|
|
}
|
|
for (const tool of proposals.tools) {
|
|
console.info(
|
|
`[eval] toolProposal variant=${tool.variantId} tool=${tool.toolName} why=${tool.whyNeeded}`,
|
|
);
|
|
}
|
|
} finally {
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "tournament") {
|
|
const { createEvalLlmClient } = await import("./runner.js");
|
|
const {
|
|
loadTournamentCases,
|
|
runTournament,
|
|
saveTournamentResult,
|
|
} = await import("./loop/tournament.js");
|
|
const variantIds = parseCsv(
|
|
getFlagString(flags, "variants") ?? positionals[0],
|
|
);
|
|
if (variantIds.length < 2) {
|
|
throw new Error(
|
|
'eval:tournament requires --variants=control,<variantId>',
|
|
);
|
|
}
|
|
|
|
const mongo = await connectMongoForEval();
|
|
try {
|
|
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
|
|
const cases = await loadTournamentCases(mongo.connection, {
|
|
caseIds: parseCsv(getFlagString(flags, "case-ids")),
|
|
limit: maxMatches,
|
|
});
|
|
const result = await runTournament(createEvalLlmClient(), {
|
|
cases,
|
|
maxMatches,
|
|
variantIds,
|
|
});
|
|
await saveTournamentResult(mongo.connection, result);
|
|
printTournamentResult(result);
|
|
} finally {
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (command === "tournament-results") {
|
|
const { queryTournamentResults } = await import("./loop/tournament.js");
|
|
const mongo = await connectMongoForEval();
|
|
try {
|
|
const results = await queryTournamentResults(mongo.connection, {
|
|
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
|
variantId: getFlagString(flags, "variant"),
|
|
});
|
|
for (const result of results) {
|
|
printTournamentResult(result);
|
|
}
|
|
} finally {
|
|
await mongo.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
throw new Error(`Unknown eval command: ${command}`);
|
|
} catch (error) {
|
|
logger.error(
|
|
{ err: error },
|
|
`[eval] "${command}" failed`,
|
|
);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
void main();
|