Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+166 -17
View File
@@ -1,9 +1,3 @@
function printPhaseNotice(command: string): void {
console.info(
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
);
}
function parsePositiveInteger(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
@@ -13,26 +7,181 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
interface ParsedArgs {
command: string;
flags: Map<string, string | true>;
positionals: string[];
}
function parseArgs(argv: string[]): ParsedArgs {
const [command = "run", ...rest] = argv;
const flags = new Map<string, string | true>();
const positionals: string[] = [];
for (let index = 0; index < rest.length; index++) {
const token = rest[index];
if (!token) {
continue;
}
if (!token.startsWith("--")) {
positionals.push(token);
continue;
}
const withoutPrefix = token.slice(2);
const equalsIndex = withoutPrefix.indexOf("=");
if (equalsIndex >= 0) {
const key = withoutPrefix.slice(0, equalsIndex);
const value = withoutPrefix.slice(equalsIndex + 1);
flags.set(key, value || true);
continue;
}
const next = rest[index + 1];
if (next && !next.startsWith("--")) {
flags.set(withoutPrefix, next);
index += 1;
continue;
}
flags.set(withoutPrefix, true);
}
return {
command,
flags,
positionals,
};
}
function getFlagString(
flags: Map<string, string | true>,
key: string,
): string | undefined {
const value = flags.get(key);
return typeof value === "string" ? value : undefined;
}
function hasFlag(
flags: Map<string, string | true>,
key: string,
): boolean {
return flags.has(key);
}
function printRunSummary(report: {
generatedAt: string;
summary: {
cases: number;
passRate: number;
ruleAverageScore: number;
averageLatencyMs: number;
llmAverageOverall: number | null;
llmJudgedCases: number;
};
baselineDelta?: {
passRate: number | null;
ruleAverageScore: number | null;
llmAverageOverall: number | null;
averageLatencyMs: number | null;
} | undefined;
}, reportPath: string): void {
const llmLine =
report.summary.llmAverageOverall === null
? "n/a"
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
console.info(`[eval] generatedAt=${report.generatedAt}`);
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
console.info(
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
);
console.info(`[eval] llmAverageOverall=${llmLine}`);
if (report.baselineDelta) {
console.info(
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
);
}
console.info(`[eval] report=${reportPath}`);
}
async function main(): Promise<void> {
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
loadEnvironmentFromNodeEnv();
const { getLogger, initLogger } = await import("@obs/logger");
initLogger();
const { captureGoldenSetScaffold } = await import("./capture.js");
const logger = getLogger();
const [command = "run", arg1, arg2] = process.argv.slice(2);
const parsed = parseArgs(process.argv.slice(2));
const { command, flags, positionals } = parsed;
if (command === "capture") {
const outputPath = await captureGoldenSetScaffold({
limit: parsePositiveInteger(arg1),
outputPath: arg2,
});
getLogger().info({ outputPath }, '[eval] "capture" completed');
return;
try {
if (command === "capture") {
const { captureGoldenSetScaffold } = await import("./capture.js");
const outputPath = await captureGoldenSetScaffold({
limit:
parsePositiveInteger(getFlagString(flags, "limit")) ??
parsePositiveInteger(positionals[0]),
outputPath: getFlagString(flags, "output") ?? positionals[1],
});
logger.info({ outputPath }, '[eval] "capture" completed');
return;
}
if (command === "run") {
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
"./runner.js"
);
const { publishEvalReportIfPossible } = await import(
"./runner.js"
);
const setName = getFlagString(flags, "set") ?? "golden";
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
if (hasFlag(flags, "dry-run")) {
const cases = await loadEvalCases(inputPath, { limit });
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
return;
}
const { report, reportPath } = await runOnboardingEval({
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
inputPath,
limit,
outputPath: getFlagString(flags, "output"),
setName,
});
const publishedReport = hasFlag(flags, "no-publish-report")
? false
: await publishEvalReportIfPossible(report);
printRunSummary(report, reportPath);
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
return;
}
if (command === "baseline") {
const { updateEvalBaseline } = await import("./runner.js");
const reportPath = getFlagString(flags, "report") ?? positionals[0];
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
console.info(
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
);
return;
}
throw new Error(`Unknown eval command: ${command}`);
} catch (error) {
logger.error(
{ err: error },
`[eval] "${command}" failed`,
);
process.exitCode = 1;
}
printPhaseNotice(command);
}
void main();