Close supervisor review loop and add handoff eval

This commit is contained in:
dahyeon committed 2026-05-18 18:04:01 -07:00
1 parent c14adac162
commit f0b00e5fd2
20 files changed
+2017 -129

No files matched your search

+170
View File
@@ -0,0 +1,170 @@
import { describe, expect, it } from "vitest";
import type { StreamEntry } from "@common/garnet";
import {
getSupervisorReview,
getSupervisorReviewStats,
listSupervisorReview,
parseSupervisorReviewEntry,
resolveSupervisorReview,
SUPERVISOR_RESOLVED_HASH_PREFIX,
SUPERVISOR_REVIEW_STREAM_KEY,
} from "./review-queue.js";
class FakeGarnet {
readonly hashes = new Map<string, Record<string, string>>();
readonly stream: StreamEntry[];
constructor(stream: StreamEntry[]) {
this.stream = stream;
}
readStreamRange(
key: string,
start: string,
end: string,
options?: { count?: number | undefined },
): Promise<StreamEntry[]> {
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
const entries = this.stream.filter((entry) => {
const afterStart = start === "-" || entry.id >= start;
const beforeEnd = end === "+" || entry.id <= end;
return afterStart && beforeEnd;
});
return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries);
}
getHash(key: string): Promise<Record<string, string>> {
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
}
setHash(key: string, fields: Record<string, string>): Promise<void> {
this.hashes.set(key, { ...fields });
return Promise.resolve();
}
expireKey(): Promise<boolean> {
return Promise.resolve(true);
}
}
const packet = {
constraints: [],
currentState: {},
evidence: [],
expectedOutputSchema: "text",
handoffReason: "out-of-scope",
sourceChain: ["onboarding", "moderation"],
summary: "user asked about ads",
userGoal: "광고 규칙 문의",
};
const entries: StreamEntry[] = [
{
id: "1-0",
message: {
chatroomId: "room-1",
packet: JSON.stringify(packet),
queuedAt: "2026-05-18T00:00:00.000Z",
reason: "out-of-scope",
requestId: "request-1",
target: "moderation",
type: "supervisor-handoff",
userId: "user-1",
},
},
{
id: "2-0",
message: {
chatroomId: "room-2",
queuedAt: "2026-05-18T01:00:00.000Z",
reason: "low-confidence",
requestId: "request-2",
target: "human-review",
type: "supervisor-handoff",
userId: "user-2",
},
},
];
describe("supervisor review queue", () => {
const firstEntry = entries[0]!;
it("parses a stream entry with the preserved handoff packet", () => {
expect(parseSupervisorReviewEntry(firstEntry)).toMatchObject({
chatroomId: "room-1",
id: "1-0",
packet,
reason: "out-of-scope",
target: "moderation",
});
});
it("lists pending entries and filters resolved items by default", async () => {
const garnet = new FakeGarnet(entries);
await resolveSupervisorReview(garnet, "1-0", {
action: "manual-reply",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await expect(listSupervisorReview(garnet)).resolves.toHaveLength(1);
await expect(
listSupervisorReview(garnet, { includeResolved: true, reason: "out-of-scope" }),
).resolves.toMatchObject([
{
id: "1-0",
resolution: {
action: "manual-reply",
resolvedBy: "operator@example.com",
},
},
]);
});
it("gets a single entry with resolution metadata", async () => {
const garnet = new FakeGarnet(entries);
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
action: "dismiss",
note: "duplicate",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await expect(getSupervisorReview(garnet, "1-0")).resolves.toMatchObject({
id: "1-0",
resolution: {
action: "dismiss",
note: "duplicate",
},
});
});
it("computes reason totals and top source chains", async () => {
const garnet = new FakeGarnet(entries);
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
action: "reroute",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await expect(getSupervisorReviewStats(garnet)).resolves.toMatchObject({
byReason: {
"low-confidence": {
pending: 1,
resolved: 0,
total: 1,
},
"out-of-scope": {
pending: 0,
resolved: 1,
total: 1,
},
},
pending: 1,
resolved: 1,
topSourceChains: [{ chain: "onboarding>moderation", count: 1 }],
total: 2,
});
});
});
+336 -6
View File
@@ -1,14 +1,344 @@
import type { GarnetClient } from "@common/garnet";
import type { HandoffPacket } from "@agent/handoff-packet";
import type { HandoffReason } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet";
import {
getSupervisorReviewQueueMaxLen,
getSupervisorReviewResolutionTtlSec,
} from "../../config.js";
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
"lolympus-agent:review-resolved:";
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
export interface SupervisorReviewEntry {
id: string;
queuedAt: string;
chatroomId: string;
userId: string;
requestId: string;
reason: HandoffReason;
target: string;
packet?: HandoffPacket | undefined;
extra?: Record<string, string> | undefined;
}
export interface SupervisorReviewResolution {
resolvedAt: string;
resolvedBy: string;
action: SupervisorReviewAction;
note?: string | undefined;
}
export interface SupervisorReviewItem extends SupervisorReviewEntry {
resolution?: SupervisorReviewResolution | undefined;
}
export interface ListSupervisorReviewOptions {
includeResolved?: boolean | undefined;
limit?: number | undefined;
reason?: HandoffReason | undefined;
since?: string | undefined;
sinceQueuedAt?: string | undefined;
until?: string | undefined;
untilQueuedAt?: string | undefined;
}
export interface SupervisorReviewReasonStats {
pending: number;
resolved: number;
total: number;
}
export interface SupervisorReviewStats {
averagePendingAgeMs: number;
byReason: Record<string, SupervisorReviewReasonStats>;
pending: number;
resolved: number;
topSourceChains: Array<{ chain: string; count: number }>;
total: number;
}
type SupervisorReviewListGarnetClient = Pick<
GarnetClient,
"getHash" | "readStreamRange"
>;
type SupervisorReviewResolveGarnetClient = Pick<
GarnetClient,
"expireKey" | "setHash"
>;
const BASE_FIELDS = new Set([
"chatroomId",
"packet",
"queuedAt",
"reason",
"requestId",
"target",
"type",
"userId",
]);
function isHandoffReason(value: string | undefined): value is HandoffReason {
return (
value === "out-of-scope" ||
value === "missing-tool" ||
value === "low-confidence" ||
value === "validation-failed" ||
value === "same-agent-revisit" ||
value === "max-hop" ||
value === "high-risk-action" ||
value === "policy-risk"
);
}
function isSupervisorReviewAction(
value: string | undefined,
): value is SupervisorReviewAction {
return value === "manual-reply" || value === "reroute" || value === "dismiss";
}
function parseOptionalPacket(value: string | undefined): HandoffPacket | undefined {
if (!value?.trim() || value.trim() === "null") {
return undefined;
}
try {
const parsed = JSON.parse(value) as unknown;
return typeof parsed === "object" && parsed !== null
? (parsed as HandoffPacket)
: undefined;
} catch {
return undefined;
}
}
function parseResolution(
fields: Record<string, string>,
): SupervisorReviewResolution | undefined {
if (
!fields["resolvedAt"] ||
!fields["resolvedBy"] ||
!isSupervisorReviewAction(fields["action"])
) {
return undefined;
}
return {
action: fields["action"],
resolvedAt: fields["resolvedAt"],
resolvedBy: fields["resolvedBy"],
...(fields["note"] ? { note: fields["note"] } : {}),
};
}
function resolutionKey(id: string): string {
return `${SUPERVISOR_RESOLVED_HASH_PREFIX}${id}`;
}
function isAfterOrEqual(value: string, lowerBound: string | undefined): boolean {
return !lowerBound || value >= lowerBound;
}
function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean {
return !upperBound || value <= upperBound;
}
export async function appendSupervisorReview(
garnetClient: Pick<GarnetClient, "appendStream">,
fields: Record<string, string>,
): Promise<void> {
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
await garnetClient.appendStream(
SUPERVISOR_REVIEW_STREAM_KEY,
{
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
},
{ maxLen: getSupervisorReviewQueueMaxLen() },
);
}
export function parseSupervisorReviewEntry(
streamEntry: StreamEntry,
): SupervisorReviewEntry | null {
const { message } = streamEntry;
const reason = message["reason"];
if (
!message["queuedAt"] ||
!message["chatroomId"] ||
!message["userId"] ||
!message["requestId"] ||
!message["target"] ||
!isHandoffReason(reason)
) {
return null;
}
const extra = Object.fromEntries(
Object.entries(message).filter(([key]) => !BASE_FIELDS.has(key)),
);
const packet = parseOptionalPacket(message["packet"]);
return {
chatroomId: message["chatroomId"],
id: streamEntry.id,
queuedAt: message["queuedAt"],
reason,
requestId: message["requestId"],
target: message["target"],
userId: message["userId"],
...(packet ? { packet } : {}),
...(Object.keys(extra).length > 0 ? { extra } : {}),
};
}
export async function getSupervisorReview(
garnetClient: SupervisorReviewListGarnetClient,
id: string,
): Promise<SupervisorReviewItem | null> {
const entries = await garnetClient.readStreamRange(
SUPERVISOR_REVIEW_STREAM_KEY,
id,
id,
{ count: 1 },
);
const parsed = entries.flatMap((entry) => {
const review = parseSupervisorReviewEntry(entry);
return review ? [review] : [];
})[0];
if (!parsed) {
return null;
}
const resolution = parseResolution(
await garnetClient.getHash(resolutionKey(parsed.id)),
);
return {
...parsed,
...(resolution ? { resolution } : {}),
};
}
export async function listSupervisorReview(
garnetClient: SupervisorReviewListGarnetClient,
options?: ListSupervisorReviewOptions,
): Promise<SupervisorReviewItem[]> {
const entries = await garnetClient.readStreamRange(
SUPERVISOR_REVIEW_STREAM_KEY,
options?.since ?? "-",
options?.until ?? "+",
);
const items: SupervisorReviewItem[] = [];
for (const entry of entries) {
const review = parseSupervisorReviewEntry(entry);
if (!review) {
continue;
}
if (options?.reason && review.reason !== options.reason) {
continue;
}
if (!isAfterOrEqual(review.queuedAt, options?.sinceQueuedAt)) {
continue;
}
if (!isBeforeOrEqual(review.queuedAt, options?.untilQueuedAt)) {
continue;
}
const resolution = parseResolution(
await garnetClient.getHash(resolutionKey(review.id)),
);
if (resolution && !options?.includeResolved) {
continue;
}
items.push({
...review,
...(resolution ? { resolution } : {}),
});
if (options?.limit && items.length >= options.limit) {
break;
}
}
return items;
}
export async function resolveSupervisorReview(
garnetClient: SupervisorReviewResolveGarnetClient,
id: string,
resolution: SupervisorReviewResolution,
): Promise<void> {
const fields: Record<string, string> = {
action: resolution.action,
resolvedAt: resolution.resolvedAt,
resolvedBy: resolution.resolvedBy,
};
if (resolution.note?.trim()) {
fields.note = resolution.note.trim();
}
const key = resolutionKey(id);
await garnetClient.setHash(key, fields);
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
}
export async function getSupervisorReviewStats(
garnetClient: SupervisorReviewListGarnetClient,
options?: Omit<ListSupervisorReviewOptions, "includeResolved">,
): Promise<SupervisorReviewStats> {
const nowMs = Date.now();
const items = await listSupervisorReview(garnetClient, {
...options,
includeResolved: true,
});
const byReason: Record<string, SupervisorReviewReasonStats> = {};
const sourceChains = new Map<string, number>();
let pendingAgeMs = 0;
let pending = 0;
let resolved = 0;
for (const item of items) {
const reasonStats = byReason[item.reason] ?? {
pending: 0,
resolved: 0,
total: 0,
};
reasonStats.total += 1;
if (item.resolution) {
resolved += 1;
reasonStats.resolved += 1;
} else {
pending += 1;
reasonStats.pending += 1;
const queuedAtMs = Date.parse(item.queuedAt);
if (Number.isFinite(queuedAtMs)) {
pendingAgeMs += Math.max(0, nowMs - queuedAtMs);
}
}
byReason[item.reason] = reasonStats;
const chain = item.packet?.sourceChain?.join(">");
if (chain) {
sourceChains.set(chain, (sourceChains.get(chain) ?? 0) + 1);
}
}
return {
averagePendingAgeMs: pending === 0 ? 0 : Math.round(pendingAgeMs / pending),
byReason,
pending,
resolved,
topSourceChains: [...sourceChains.entries()]
.map(([chain, count]) => ({ chain, count }))
.sort((left, right) => right.count - left.count)
.slice(0, 5),
total: items.length,
};
}
+9
View File
@@ -82,6 +82,15 @@ describe("SupervisorSkill", () => {
type: "supervisor-handoff",
},
});
expect(garnet.streams[0]?.fields["packet"]).toEqual(expect.any(String));
const queuedPacket: unknown = JSON.parse(
String(garnet.streams[0]?.fields["packet"]),
);
expect(queuedPacket).toMatchObject({
handoffReason: "max-hop",
sourceChain: ["onboarding", "supervisor"],
userGoal: "가입 상태 확인",
});
expect(nats.published[0]).toMatchObject({
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
+1
View File
@@ -56,6 +56,7 @@ export class SupervisorSkill implements Skill {
await appendSupervisorReview(this.garnetClient, {
chatroomId: request.chatroomId,
packet: JSON.stringify(packet ?? null),
reason,
requestId: request.requestId,
target: target ?? "human-review",