Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||
import { publishEvalReportIfPossible } from "./runner.js";
|
||||
import type { EvalReport } from "./types.js";
|
||||
|
||||
const natsMocks = vi.hoisted(() => ({
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
connect: vi.fn().mockResolvedValue(undefined),
|
||||
publish: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@common/nats", () => {
|
||||
return {
|
||||
NatsClient: class {
|
||||
connect = natsMocks.connect;
|
||||
publish = natsMocks.publish;
|
||||
close = natsMocks.close;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const report: EvalReport = {
|
||||
generatedAt: "2026-04-17T00:00:00.000Z",
|
||||
inputPath: "/tmp/onboarding.jsonl",
|
||||
setName: "golden",
|
||||
summary: {
|
||||
averageLatencyMs: 100,
|
||||
cases: 1,
|
||||
emptyResponses: 0,
|
||||
llmAverageAccuracy: 4,
|
||||
llmAverageConcision: 4,
|
||||
llmAverageFriendliness: 5,
|
||||
llmAverageOverall: 4.25,
|
||||
llmAverageScope: 4,
|
||||
llmJudgedCases: 1,
|
||||
passRate: 1,
|
||||
passedCases: 1,
|
||||
ruleAverageScore: 1,
|
||||
},
|
||||
cases: [],
|
||||
};
|
||||
|
||||
describe("publishEvalReportIfPossible", () => {
|
||||
it("publishes the report to the eval report subject", async () => {
|
||||
const result = await publishEvalReportIfPossible(report);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(natsMocks.connect).toHaveBeenCalledTimes(1);
|
||||
expect(natsMocks.publish).toHaveBeenCalledWith(
|
||||
AGENT_EVAL_REPORT_SUBJECT,
|
||||
report,
|
||||
);
|
||||
expect(natsMocks.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user