136 lines
3.5 KiB
TypeScript
136 lines
3.5 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import type { StreamEntry } from "@common/garnet";
|
|
import {
|
|
aggregateRecentEvalResults,
|
|
EVAL_AGG_CURSOR_KEY,
|
|
EVAL_RESULTS_STREAM_KEY,
|
|
getSliceAggregateKey,
|
|
} from "./aggregator.js";
|
|
|
|
class MemoryGarnet {
|
|
readonly expires = new Map<string, number>();
|
|
readonly hashes = new Map<string, Record<string, string>>();
|
|
readonly strings = new Map<string, string>();
|
|
readonly streams = new Map<string, StreamEntry[]>();
|
|
|
|
expireKey(key: string, ttlSeconds: number): Promise<boolean> {
|
|
this.expires.set(key, ttlSeconds);
|
|
return Promise.resolve(true);
|
|
}
|
|
|
|
getHash(key: string): Promise<Record<string, string>> {
|
|
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
|
|
}
|
|
|
|
readStreamRange(
|
|
key: string,
|
|
start: string,
|
|
_end: string,
|
|
): Promise<StreamEntry[]> {
|
|
const entries = this.streams.get(key) ?? [];
|
|
if (!start.startsWith("(")) {
|
|
return Promise.resolve(entries);
|
|
}
|
|
|
|
const exclusiveStart = start.slice(1);
|
|
return Promise.resolve(entries.filter((entry) => entry.id > exclusiveStart));
|
|
}
|
|
|
|
scanKeys(pattern: string): Promise<string[]> {
|
|
const prefix = pattern.replace(/\*$/u, "");
|
|
return Promise.resolve(
|
|
[...this.hashes.keys()].filter((key) => key.startsWith(prefix)),
|
|
);
|
|
}
|
|
|
|
setHash(key: string, fields: Record<string, string>): Promise<void> {
|
|
this.hashes.set(key, { ...fields });
|
|
return Promise.resolve();
|
|
}
|
|
|
|
setString(key: string, value: string): Promise<void> {
|
|
this.strings.set(key, value);
|
|
return Promise.resolve();
|
|
}
|
|
}
|
|
|
|
function resultEntry(
|
|
id: string,
|
|
fields: Partial<Record<string, string>>,
|
|
): StreamEntry {
|
|
return {
|
|
id,
|
|
message: {
|
|
type: "shadow-eval-result",
|
|
tier: "0",
|
|
intent: "status-question",
|
|
latencyMs: "100",
|
|
llmOverall: "4",
|
|
validatorStatus: "pass",
|
|
derived: JSON.stringify({
|
|
tier: 0,
|
|
intent: { kind: "status-question" },
|
|
phase: { kind: "photo-pending" },
|
|
}),
|
|
...fields,
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("aggregateRecentEvalResults", () => {
|
|
it("rolls stream entries up into per-slice hashes", async () => {
|
|
const garnet = new MemoryGarnet();
|
|
garnet.streams.set(EVAL_RESULTS_STREAM_KEY, [
|
|
resultEntry("1-0", {}),
|
|
resultEntry("2-0", {
|
|
latencyMs: "300",
|
|
llmOverall: "",
|
|
llmJudge: JSON.stringify({
|
|
skipped: false,
|
|
scores: { overall: 2 },
|
|
}),
|
|
validatorStatus: "reject",
|
|
}),
|
|
resultEntry("3-0", {
|
|
tier: "1",
|
|
intent: "photo-question",
|
|
latencyMs: "",
|
|
llmOverall: "",
|
|
validatorStatus: "soft-warn",
|
|
derived: JSON.stringify({
|
|
tier: 1,
|
|
intent: { kind: "photo-question" },
|
|
phase: { kind: "fresh" },
|
|
}),
|
|
}),
|
|
]);
|
|
|
|
const result = await aggregateRecentEvalResults(garnet, {
|
|
sinceId: "0-0",
|
|
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
|
});
|
|
|
|
expect(result).toEqual({ lastSeenId: "3-0", updatedSlices: 2 });
|
|
expect(garnet.strings.get(EVAL_AGG_CURSOR_KEY)).toBe("3-0");
|
|
|
|
const key = getSliceAggregateKey({
|
|
tier: 0,
|
|
intent: "status-question",
|
|
phase: "photo-pending",
|
|
});
|
|
expect(garnet.hashes.get(key)).toMatchObject({
|
|
count: "2",
|
|
llmOverallSum: "6",
|
|
llmOverallCount: "2",
|
|
validatorRejectCount: "1",
|
|
latencySumMs: "400",
|
|
});
|
|
expect(JSON.parse(garnet.hashes.get(key)?.["latencySamples"] ?? "[]")).toEqual([
|
|
100,
|
|
300,
|
|
]);
|
|
expect(garnet.expires.get(key)).toBeGreaterThan(0);
|
|
});
|
|
});
|