Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
+166
-17
@@ -1,9 +1,3 @@
|
||||
function printPhaseNotice(command: string): void {
|
||||
console.info(
|
||||
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
@@ -13,26 +7,181 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
interface ParsedArgs {
|
||||
command: string;
|
||||
flags: Map<string, string | true>;
|
||||
positionals: string[];
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ParsedArgs {
|
||||
const [command = "run", ...rest] = argv;
|
||||
const flags = new Map<string, string | true>();
|
||||
const positionals: string[] = [];
|
||||
|
||||
for (let index = 0; index < rest.length; index++) {
|
||||
const token = rest[index];
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!token.startsWith("--")) {
|
||||
positionals.push(token);
|
||||
continue;
|
||||
}
|
||||
|
||||
const withoutPrefix = token.slice(2);
|
||||
const equalsIndex = withoutPrefix.indexOf("=");
|
||||
if (equalsIndex >= 0) {
|
||||
const key = withoutPrefix.slice(0, equalsIndex);
|
||||
const value = withoutPrefix.slice(equalsIndex + 1);
|
||||
flags.set(key, value || true);
|
||||
continue;
|
||||
}
|
||||
|
||||
const next = rest[index + 1];
|
||||
if (next && !next.startsWith("--")) {
|
||||
flags.set(withoutPrefix, next);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
flags.set(withoutPrefix, true);
|
||||
}
|
||||
|
||||
return {
|
||||
command,
|
||||
flags,
|
||||
positionals,
|
||||
};
|
||||
}
|
||||
|
||||
function getFlagString(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): string | undefined {
|
||||
const value = flags.get(key);
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function hasFlag(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): boolean {
|
||||
return flags.has(key);
|
||||
}
|
||||
|
||||
function printRunSummary(report: {
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
cases: number;
|
||||
passRate: number;
|
||||
ruleAverageScore: number;
|
||||
averageLatencyMs: number;
|
||||
llmAverageOverall: number | null;
|
||||
llmJudgedCases: number;
|
||||
};
|
||||
baselineDelta?: {
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
averageLatencyMs: number | null;
|
||||
} | undefined;
|
||||
}, reportPath: string): void {
|
||||
const llmLine =
|
||||
report.summary.llmAverageOverall === null
|
||||
? "n/a"
|
||||
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
|
||||
|
||||
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
||||
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
|
||||
console.info(
|
||||
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
|
||||
);
|
||||
console.info(`[eval] llmAverageOverall=${llmLine}`);
|
||||
|
||||
if (report.baselineDelta) {
|
||||
console.info(
|
||||
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.info(`[eval] report=${reportPath}`);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
|
||||
loadEnvironmentFromNodeEnv();
|
||||
|
||||
const { getLogger, initLogger } = await import("@obs/logger");
|
||||
initLogger();
|
||||
const { captureGoldenSetScaffold } = await import("./capture.js");
|
||||
const logger = getLogger();
|
||||
|
||||
const [command = "run", arg1, arg2] = process.argv.slice(2);
|
||||
const parsed = parseArgs(process.argv.slice(2));
|
||||
const { command, flags, positionals } = parsed;
|
||||
|
||||
if (command === "capture") {
|
||||
const outputPath = await captureGoldenSetScaffold({
|
||||
limit: parsePositiveInteger(arg1),
|
||||
outputPath: arg2,
|
||||
});
|
||||
getLogger().info({ outputPath }, '[eval] "capture" completed');
|
||||
return;
|
||||
try {
|
||||
if (command === "capture") {
|
||||
const { captureGoldenSetScaffold } = await import("./capture.js");
|
||||
const outputPath = await captureGoldenSetScaffold({
|
||||
limit:
|
||||
parsePositiveInteger(getFlagString(flags, "limit")) ??
|
||||
parsePositiveInteger(positionals[0]),
|
||||
outputPath: getFlagString(flags, "output") ?? positionals[1],
|
||||
});
|
||||
logger.info({ outputPath }, '[eval] "capture" completed');
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "run") {
|
||||
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const { publishEvalReportIfPossible } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const setName = getFlagString(flags, "set") ?? "golden";
|
||||
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
|
||||
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
|
||||
|
||||
if (hasFlag(flags, "dry-run")) {
|
||||
const cases = await loadEvalCases(inputPath, { limit });
|
||||
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const { report, reportPath } = await runOnboardingEval({
|
||||
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
|
||||
inputPath,
|
||||
limit,
|
||||
outputPath: getFlagString(flags, "output"),
|
||||
setName,
|
||||
});
|
||||
const publishedReport = hasFlag(flags, "no-publish-report")
|
||||
? false
|
||||
: await publishEvalReportIfPossible(report);
|
||||
printRunSummary(report, reportPath);
|
||||
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "baseline") {
|
||||
const { updateEvalBaseline } = await import("./runner.js");
|
||||
const reportPath = getFlagString(flags, "report") ?? positionals[0];
|
||||
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
|
||||
console.info(
|
||||
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown eval command: ${command}`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ err: error },
|
||||
`[eval] "${command}" failed`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
printPhaseNotice(command);
|
||||
}
|
||||
|
||||
void main();
|
||||
Reference in new issue
Block a user