Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+56
View File
@@ -0,0 +1,56 @@
import { describe, expect, it, vi } from "vitest";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { publishEvalReportIfPossible } from "./runner.js";
import type { EvalReport } from "./types.js";
const natsMocks = vi.hoisted(() => ({
close: vi.fn().mockResolvedValue(undefined),
connect: vi.fn().mockResolvedValue(undefined),
publish: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@common/nats", () => {
return {
NatsClient: class {
connect = natsMocks.connect;
publish = natsMocks.publish;
close = natsMocks.close;
},
};
});
const report: EvalReport = {
generatedAt: "2026-04-17T00:00:00.000Z",
inputPath: "/tmp/onboarding.jsonl",
setName: "golden",
summary: {
averageLatencyMs: 100,
cases: 1,
emptyResponses: 0,
llmAverageAccuracy: 4,
llmAverageConcision: 4,
llmAverageFriendliness: 5,
llmAverageOverall: 4.25,
llmAverageScope: 4,
llmJudgedCases: 1,
passRate: 1,
passedCases: 1,
ruleAverageScore: 1,
},
cases: [],
};
describe("publishEvalReportIfPossible", () => {
it("publishes the report to the eval report subject", async () => {
const result = await publishEvalReportIfPossible(report);
expect(result).toBe(true);
expect(natsMocks.connect).toHaveBeenCalledTimes(1);
expect(natsMocks.publish).toHaveBeenCalledWith(
AGENT_EVAL_REPORT_SUBJECT,
report,
);
expect(natsMocks.close).toHaveBeenCalledTimes(1);
});
});