Files
lolympus-agent/src/eval/router-training-signals.test.ts
T

109 lines
3.0 KiB
TypeScript

import { describe, expect, it } from "vitest";
import type { StreamEntry } from "@common/garnet";
import { BACKGROUND_STREAM_KEY } from "./shadow-bus.js";
import { loadSupervisorOutcomes } from "./router-training-signals.js";
import {
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
SUPERVISOR_REVIEW_STREAM_KEY,
} from "@skills/supervisor/review-queue";
class FakeGarnet {
constructor(private readonly streams: Record<string, StreamEntry[]>) {}
readStreamRange(key: string): Promise<StreamEntry[]> {
return Promise.resolve(this.streams[key] ?? []);
}
}
describe("loadSupervisorOutcomes", () => {
it("joins supervisor review entries, resolution audit, and shadow eval by requestId", async () => {
const garnet = new FakeGarnet({
[BACKGROUND_STREAM_KEY]: [
{
id: "10-0",
message: {
classified: "{}",
createdAt: "2026-05-18T00:00:30.000Z",
derived: "{}",
input: JSON.stringify({
request: {
chatroomId: "room-1",
nickname: "tester",
requestId: "request-1",
skill: "onboarding",
trigger: "user-message",
userId: "user-1",
},
}),
output: JSON.stringify({
confidence: 1,
requestId: "request-1",
source: "template",
text: "ok",
}),
skill: "onboarding",
},
},
],
[SUPERVISOR_RESOLUTIONS_STREAM_KEY]: [
{
id: "2-0",
message: {
action: "manual-reply",
resolvedAt: "2026-05-18T00:02:00.000Z",
resolvedBy: "operator@example.com",
reviewId: "1-0",
type: "resolution",
},
},
],
[SUPERVISOR_REVIEW_STREAM_KEY]: [
{
id: "1-0",
message: {
chatroomId: "room-1",
packet: JSON.stringify({
constraints: [],
currentState: {},
evidence: [],
expectedOutputSchema: "text",
handoffReason: "low-confidence",
sourceChain: ["onboarding", "supervisor"],
summary: "unclear",
userGoal: "help",
}),
queuedAt: "2026-05-18T00:01:00.000Z",
reason: "low-confidence",
requestId: "request-1",
target: "human-review",
userId: "user-1",
},
},
],
});
await expect(
loadSupervisorOutcomes(
garnet,
new Date("2026-05-18T00:00:00.000Z"),
new Date("2026-05-18T00:03:00.000Z"),
),
).resolves.toMatchObject([
{
reason: "low-confidence",
resolution: {
action: "manual-reply",
},
reviewId: "1-0",
shadowEvalEntry: {
request: {
requestId: "request-1",
},
},
sourceChain: ["onboarding", "supervisor"],
},
]);
});
});