Files
lolympus-agent/src/eval/aggregator.test.ts
T

136 lines
3.5 KiB
TypeScript

import { describe, expect, it } from "vitest";
import type { StreamEntry } from "@common/garnet";
import {
aggregateRecentEvalResults,
EVAL_AGG_CURSOR_KEY,
EVAL_RESULTS_STREAM_KEY,
getSliceAggregateKey,
} from "./aggregator.js";
class MemoryGarnet {
readonly expires = new Map<string, number>();
readonly hashes = new Map<string, Record<string, string>>();
readonly strings = new Map<string, string>();
readonly streams = new Map<string, StreamEntry[]>();
expireKey(key: string, ttlSeconds: number): Promise<boolean> {
this.expires.set(key, ttlSeconds);
return Promise.resolve(true);
}
getHash(key: string): Promise<Record<string, string>> {
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
}
readStreamRange(
key: string,
start: string,
_end: string,
): Promise<StreamEntry[]> {
const entries = this.streams.get(key) ?? [];
if (!start.startsWith("(")) {
return Promise.resolve(entries);
}
const exclusiveStart = start.slice(1);
return Promise.resolve(entries.filter((entry) => entry.id > exclusiveStart));
}
scanKeys(pattern: string): Promise<string[]> {
const prefix = pattern.replace(/\*$/u, "");
return Promise.resolve(
[...this.hashes.keys()].filter((key) => key.startsWith(prefix)),
);
}
setHash(key: string, fields: Record<string, string>): Promise<void> {
this.hashes.set(key, { ...fields });
return Promise.resolve();
}
setString(key: string, value: string): Promise<void> {
this.strings.set(key, value);
return Promise.resolve();
}
}
function resultEntry(
id: string,
fields: Partial<Record<string, string>>,
): StreamEntry {
return {
id,
message: {
type: "shadow-eval-result",
tier: "0",
intent: "status-question",
latencyMs: "100",
llmOverall: "4",
validatorStatus: "pass",
derived: JSON.stringify({
tier: 0,
intent: { kind: "status-question" },
phase: { kind: "photo-pending" },
}),
...fields,
},
};
}
describe("aggregateRecentEvalResults", () => {
it("rolls stream entries up into per-slice hashes", async () => {
const garnet = new MemoryGarnet();
garnet.streams.set(EVAL_RESULTS_STREAM_KEY, [
resultEntry("1-0", {}),
resultEntry("2-0", {
latencyMs: "300",
llmOverall: "",
llmJudge: JSON.stringify({
skipped: false,
scores: { overall: 2 },
}),
validatorStatus: "reject",
}),
resultEntry("3-0", {
tier: "1",
intent: "photo-question",
latencyMs: "",
llmOverall: "",
validatorStatus: "soft-warn",
derived: JSON.stringify({
tier: 1,
intent: { kind: "photo-question" },
phase: { kind: "fresh" },
}),
}),
]);
const result = await aggregateRecentEvalResults(garnet, {
sinceId: "0-0",
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
});
expect(result).toEqual({ lastSeenId: "3-0", updatedSlices: 2 });
expect(garnet.strings.get(EVAL_AGG_CURSOR_KEY)).toBe("3-0");
const key = getSliceAggregateKey({
tier: 0,
intent: "status-question",
phase: "photo-pending",
});
expect(garnet.hashes.get(key)).toMatchObject({
count: "2",
llmOverallSum: "6",
llmOverallCount: "2",
validatorRejectCount: "1",
latencySumMs: "400",
});
expect(JSON.parse(garnet.hashes.get(key)?.["latencySamples"] ?? "[]")).toEqual([
100,
300,
]);
expect(garnet.expires.get(key)).toBeGreaterThan(0);
});
});