Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
+397
@@ -1,3 +1,7 @@
|
||||
import type { MongoConnectionLike } from "./archive.js";
|
||||
import type { SliceKey, SliceReportRow } from "./aggregator.js";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
|
||||
function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
@@ -7,6 +11,78 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parsePositiveNumber(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseDurationMs(value: string | undefined): number {
|
||||
const raw = value?.trim().toLowerCase() || "7d";
|
||||
const match = raw.match(/^(\d+)(h|d)$/u);
|
||||
if (!match) {
|
||||
throw new Error("--since must use a duration like 24h, 7d, or 30d");
|
||||
}
|
||||
|
||||
const amount = Number(match[1]);
|
||||
const unit = match[2];
|
||||
if (!Number.isInteger(amount) || amount <= 0) {
|
||||
throw new Error("--since must be a positive duration");
|
||||
}
|
||||
|
||||
return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
function parseCsv(value: string | undefined): string[] {
|
||||
if (!value?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return value
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function parseSlice(value: string | undefined): SliceKey {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(
|
||||
"--slice must use tier=0,intent=status-question,phase=photo-pending",
|
||||
);
|
||||
}
|
||||
|
||||
const fields: Record<string, string> = {};
|
||||
for (const pair of value.split(",")) {
|
||||
const [key, ...rest] = pair.split("=");
|
||||
const normalizedKey = key?.trim();
|
||||
if (normalizedKey) {
|
||||
fields[normalizedKey] = rest.join("=").trim();
|
||||
}
|
||||
}
|
||||
const tier = Number(fields["tier"]);
|
||||
const intent = fields["intent"];
|
||||
const phase = fields["phase"];
|
||||
|
||||
if (
|
||||
(tier !== 0 && tier !== 1 && tier !== 2) ||
|
||||
!intent?.trim() ||
|
||||
!phase?.trim()
|
||||
) {
|
||||
throw new Error(
|
||||
"--slice must include tier=0|1|2,intent=<kind>,phase=<kind>",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
tier,
|
||||
intent,
|
||||
phase,
|
||||
};
|
||||
}
|
||||
|
||||
interface ParsedArgs {
|
||||
command: string;
|
||||
flags: Map<string, string | true>;
|
||||
@@ -108,6 +184,122 @@ function printRunSummary(report: {
|
||||
console.info(`[eval] report=${reportPath}`);
|
||||
}
|
||||
|
||||
function formatNullable(value: number | null): string {
|
||||
return value === null ? "n/a" : String(value);
|
||||
}
|
||||
|
||||
function printSliceRows(rows: SliceReportRow[]): void {
|
||||
console.info(
|
||||
[
|
||||
"tier",
|
||||
"intent",
|
||||
"phase",
|
||||
"count",
|
||||
"llmOverallAvg",
|
||||
"validatorRejectRate",
|
||||
"p50LatencyMs",
|
||||
"p95LatencyMs",
|
||||
"updatedAt",
|
||||
].join("\t"),
|
||||
);
|
||||
|
||||
for (const row of rows) {
|
||||
console.info(
|
||||
[
|
||||
row.slice.tier,
|
||||
row.slice.intent,
|
||||
row.slice.phase,
|
||||
row.count,
|
||||
formatNullable(row.llmOverallAvg),
|
||||
row.validatorRejectRate,
|
||||
formatNullable(row.p50LatencyMs),
|
||||
formatNullable(row.p95LatencyMs),
|
||||
row.updatedAt,
|
||||
].join("\t"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function printCalibrationReport(report: {
|
||||
generatedAt: string;
|
||||
meanGapOverall: number;
|
||||
pairs: number;
|
||||
pearsonByDim: {
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
friendliness: number;
|
||||
scope: number;
|
||||
};
|
||||
pearsonOverall: number;
|
||||
worstDisagreements: Array<{
|
||||
evalStreamId: string;
|
||||
humanScore: number;
|
||||
llmOverall: number;
|
||||
}>;
|
||||
}): void {
|
||||
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
||||
console.info(`[eval] pairs=${report.pairs}`);
|
||||
console.info(`[eval] pearsonOverall=${report.pearsonOverall}`);
|
||||
console.info(`[eval] meanGapOverall=${report.meanGapOverall}`);
|
||||
console.info(
|
||||
`[eval] pearsonByDim friendliness=${report.pearsonByDim.friendliness} accuracy=${report.pearsonByDim.accuracy} concision=${report.pearsonByDim.concision} scope=${report.pearsonByDim.scope}`,
|
||||
);
|
||||
for (const item of report.worstDisagreements) {
|
||||
console.info(
|
||||
`[eval] disagreement evalStreamId=${item.evalStreamId} human=${item.humanScore} llm=${item.llmOverall}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function printTournamentResult(result: {
|
||||
generatedAt: string;
|
||||
matches: Array<{ judgeVerdict: string; variantA: string; variantB: string }>;
|
||||
variants: Array<{ elo: number; id: string; matches: number; wins: number }>;
|
||||
}): void {
|
||||
console.info(`[eval] generatedAt=${result.generatedAt}`);
|
||||
console.info(`[eval] matches=${result.matches.length}`);
|
||||
for (const variant of result.variants) {
|
||||
console.info(
|
||||
`[eval] variant id=${variant.id} elo=${variant.elo} matches=${variant.matches} wins=${variant.wins}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function connectMongoForEval(): Promise<{
|
||||
close: () => Promise<void>;
|
||||
connection: MongoConnectionLike;
|
||||
}> {
|
||||
const { initMongoDB, closeMongoDB } = await import("@common/mongodb");
|
||||
const { getSecret } = await import("@common/secret-manager");
|
||||
const connectionString =
|
||||
getSecret("MONGODB_CONNECTION_STRING") ??
|
||||
process.env["MONGODB_CONNECTION_STRING"];
|
||||
|
||||
if (!connectionString) {
|
||||
throw new Error(
|
||||
"Missing MONGODB_CONNECTION_STRING for eval command. Set it in the environment or secrets.",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
connection: await initMongoDB(connectionString),
|
||||
close: closeMongoDB,
|
||||
};
|
||||
}
|
||||
|
||||
async function connectGarnetForEval(): Promise<{
|
||||
close: () => Promise<void>;
|
||||
garnetClient: GarnetClient;
|
||||
}> {
|
||||
const { GarnetClient } = await import("@common/garnet");
|
||||
const garnetClient = new GarnetClient();
|
||||
await garnetClient.connect();
|
||||
return {
|
||||
garnetClient,
|
||||
close: () => garnetClient.close(),
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
|
||||
loadEnvironmentFromNodeEnv();
|
||||
@@ -174,6 +366,211 @@ async function main(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "report") {
|
||||
const { queryArchiveSliceReportRows } = await import("./archive.js");
|
||||
const limit =
|
||||
parsePositiveInteger(getFlagString(flags, "worst")) ??
|
||||
parsePositiveInteger(getFlagString(flags, "top")) ??
|
||||
20;
|
||||
const sort = hasFlag(flags, "worst") ? "score-asc" : "count-desc";
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const rows = await queryArchiveSliceReportRows(mongo.connection, {
|
||||
limit,
|
||||
sinceMs: parseDurationMs(getFlagString(flags, "since")),
|
||||
sort,
|
||||
});
|
||||
printSliceRows(rows);
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "promote") {
|
||||
const { appendCandidatesToJsonl, selectPromotionCandidates } = await import(
|
||||
"./promote.js"
|
||||
);
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const candidates = await selectPromotionCandidates(mongo.connection, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")),
|
||||
});
|
||||
|
||||
if (hasFlag(flags, "dry-run")) {
|
||||
for (const candidate of candidates) {
|
||||
const overall = candidate.llmJudge.skipped
|
||||
? "n/a"
|
||||
: candidate.llmJudge.scores.overall;
|
||||
console.info(
|
||||
`[eval] candidate id=${candidate.caseId} reason=${candidate.reason} overall=${overall} trigger=${candidate.request.trigger}`,
|
||||
);
|
||||
}
|
||||
console.info(`[eval] dryRunCandidates=${candidates.length}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await appendCandidatesToJsonl(
|
||||
candidates,
|
||||
getFlagString(flags, "output"),
|
||||
);
|
||||
console.info(
|
||||
`[eval] promoted appended=${result.appendedCount} output=${result.outputPath}`,
|
||||
);
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "review") {
|
||||
const { parseReviewReason, runReviewCli } = await import("./review/cli.js");
|
||||
const garnet = await connectGarnetForEval();
|
||||
try {
|
||||
const result = await runReviewCli(garnet.garnetClient, {
|
||||
exportLabeled: getFlagString(flags, "export-labeled"),
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
reason: parseReviewReason(getFlagString(flags, "reason")),
|
||||
resetPending: hasFlag(flags, "reset-pending"),
|
||||
});
|
||||
console.info(
|
||||
`[eval] review completed labeled=${result.labeled} skipped=${result.skipped}`,
|
||||
);
|
||||
} finally {
|
||||
await garnet.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "calibrate") {
|
||||
const { getEvalCalibrationMinCorr } = await import("../config.js");
|
||||
const { computeCalibrationReport } = await import(
|
||||
"./review/calibration.js"
|
||||
);
|
||||
const mongo = await connectMongoForEval();
|
||||
const garnet = await connectGarnetForEval();
|
||||
try {
|
||||
const report = await computeCalibrationReport(
|
||||
mongo.connection,
|
||||
garnet.garnetClient,
|
||||
{
|
||||
sinceMs: getFlagString(flags, "since")
|
||||
? parseDurationMs(getFlagString(flags, "since"))
|
||||
: undefined,
|
||||
},
|
||||
);
|
||||
printCalibrationReport(report);
|
||||
if (report.pairs < 10) {
|
||||
console.warn(
|
||||
"[eval] calibration pairs<10; check gate is informational until at least 10 labels are available",
|
||||
);
|
||||
}
|
||||
if (
|
||||
hasFlag(flags, "check") &&
|
||||
report.pairs >= 10 &&
|
||||
report.pearsonOverall < getEvalCalibrationMinCorr()
|
||||
) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} finally {
|
||||
await garnet.close();
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "propose") {
|
||||
const { createEvalLlmClient } = await import("./runner.js");
|
||||
const {
|
||||
buildProposalInputFromArchive,
|
||||
proposeVariants,
|
||||
writePromptVariantProposal,
|
||||
} = await import("./loop/propose.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const input = await buildProposalInputFromArchive(
|
||||
mongo.connection,
|
||||
parseSlice(getFlagString(flags, "slice")),
|
||||
{
|
||||
force: hasFlag(flags, "force"),
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
},
|
||||
);
|
||||
const proposals = await proposeVariants(createEvalLlmClient(), input);
|
||||
for (const proposal of proposals.prompts) {
|
||||
const outputPath = await writePromptVariantProposal(
|
||||
proposal,
|
||||
getFlagString(flags, "output-dir"),
|
||||
);
|
||||
console.info(
|
||||
`[eval] proposal variant=${proposal.variantId} output=${outputPath}`,
|
||||
);
|
||||
}
|
||||
for (const tool of proposals.tools) {
|
||||
console.info(
|
||||
`[eval] toolProposal variant=${tool.variantId} tool=${tool.toolName} why=${tool.whyNeeded}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "tournament") {
|
||||
const { createEvalLlmClient } = await import("./runner.js");
|
||||
const {
|
||||
loadTournamentCases,
|
||||
runTournament,
|
||||
saveTournamentResult,
|
||||
} = await import("./loop/tournament.js");
|
||||
const variantIds = parseCsv(
|
||||
getFlagString(flags, "variants") ?? positionals[0],
|
||||
);
|
||||
if (variantIds.length < 2) {
|
||||
throw new Error(
|
||||
'eval:tournament requires --variants=control,<variantId>',
|
||||
);
|
||||
}
|
||||
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
|
||||
const cases = await loadTournamentCases(mongo.connection, {
|
||||
caseIds: parseCsv(getFlagString(flags, "case-ids")),
|
||||
limit: maxMatches,
|
||||
});
|
||||
const result = await runTournament(createEvalLlmClient(), {
|
||||
cases,
|
||||
maxMatches,
|
||||
variantIds,
|
||||
});
|
||||
await saveTournamentResult(mongo.connection, result);
|
||||
printTournamentResult(result);
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "tournament-results") {
|
||||
const { queryTournamentResults } = await import("./loop/tournament.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const results = await queryTournamentResults(mongo.connection, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
variantId: getFlagString(flags, "variant"),
|
||||
});
|
||||
for (const result of results) {
|
||||
printTournamentResult(result);
|
||||
}
|
||||
} finally {
|
||||
await mongo.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown eval command: ${command}`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
|
||||
Reference in new issue
Block a user