Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -0,0 +1,135 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
aggregateRecentEvalResults,
|
||||
EVAL_AGG_CURSOR_KEY,
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
getSliceAggregateKey,
|
||||
} from "./aggregator.js";
|
||||
|
||||
class MemoryGarnet {
|
||||
readonly expires = new Map<string, number>();
|
||||
readonly hashes = new Map<string, Record<string, string>>();
|
||||
readonly strings = new Map<string, string>();
|
||||
readonly streams = new Map<string, StreamEntry[]>();
|
||||
|
||||
expireKey(key: string, ttlSeconds: number): Promise<boolean> {
|
||||
this.expires.set(key, ttlSeconds);
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
|
||||
getHash(key: string): Promise<Record<string, string>> {
|
||||
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
|
||||
}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
_end: string,
|
||||
): Promise<StreamEntry[]> {
|
||||
const entries = this.streams.get(key) ?? [];
|
||||
if (!start.startsWith("(")) {
|
||||
return Promise.resolve(entries);
|
||||
}
|
||||
|
||||
const exclusiveStart = start.slice(1);
|
||||
return Promise.resolve(entries.filter((entry) => entry.id > exclusiveStart));
|
||||
}
|
||||
|
||||
scanKeys(pattern: string): Promise<string[]> {
|
||||
const prefix = pattern.replace(/\*$/u, "");
|
||||
return Promise.resolve(
|
||||
[...this.hashes.keys()].filter((key) => key.startsWith(prefix)),
|
||||
);
|
||||
}
|
||||
|
||||
setHash(key: string, fields: Record<string, string>): Promise<void> {
|
||||
this.hashes.set(key, { ...fields });
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
setString(key: string, value: string): Promise<void> {
|
||||
this.strings.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
function resultEntry(
|
||||
id: string,
|
||||
fields: Partial<Record<string, string>>,
|
||||
): StreamEntry {
|
||||
return {
|
||||
id,
|
||||
message: {
|
||||
type: "shadow-eval-result",
|
||||
tier: "0",
|
||||
intent: "status-question",
|
||||
latencyMs: "100",
|
||||
llmOverall: "4",
|
||||
validatorStatus: "pass",
|
||||
derived: JSON.stringify({
|
||||
tier: 0,
|
||||
intent: { kind: "status-question" },
|
||||
phase: { kind: "photo-pending" },
|
||||
}),
|
||||
...fields,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("aggregateRecentEvalResults", () => {
|
||||
it("rolls stream entries up into per-slice hashes", async () => {
|
||||
const garnet = new MemoryGarnet();
|
||||
garnet.streams.set(EVAL_RESULTS_STREAM_KEY, [
|
||||
resultEntry("1-0", {}),
|
||||
resultEntry("2-0", {
|
||||
latencyMs: "300",
|
||||
llmOverall: "",
|
||||
llmJudge: JSON.stringify({
|
||||
skipped: false,
|
||||
scores: { overall: 2 },
|
||||
}),
|
||||
validatorStatus: "reject",
|
||||
}),
|
||||
resultEntry("3-0", {
|
||||
tier: "1",
|
||||
intent: "photo-question",
|
||||
latencyMs: "",
|
||||
llmOverall: "",
|
||||
validatorStatus: "soft-warn",
|
||||
derived: JSON.stringify({
|
||||
tier: 1,
|
||||
intent: { kind: "photo-question" },
|
||||
phase: { kind: "fresh" },
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
const result = await aggregateRecentEvalResults(garnet, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result).toEqual({ lastSeenId: "3-0", updatedSlices: 2 });
|
||||
expect(garnet.strings.get(EVAL_AGG_CURSOR_KEY)).toBe("3-0");
|
||||
|
||||
const key = getSliceAggregateKey({
|
||||
tier: 0,
|
||||
intent: "status-question",
|
||||
phase: "photo-pending",
|
||||
});
|
||||
expect(garnet.hashes.get(key)).toMatchObject({
|
||||
count: "2",
|
||||
llmOverallSum: "6",
|
||||
llmOverallCount: "2",
|
||||
validatorRejectCount: "1",
|
||||
latencySumMs: "400",
|
||||
});
|
||||
expect(JSON.parse(garnet.hashes.get(key)?.["latencySamples"] ?? "[]")).toEqual([
|
||||
100,
|
||||
300,
|
||||
]);
|
||||
expect(garnet.expires.get(key)).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user