Close supervisor review loop and add handoff eval
This commit is contained in:
1 parent
c14adac162
commit
f0b00e5fd2
20 files changed
+2017
-129
No files matched your search
@@ -0,0 +1,170 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorReview,
|
||||
getSupervisorReviewStats,
|
||||
listSupervisorReview,
|
||||
parseSupervisorReviewEntry,
|
||||
resolveSupervisorReview,
|
||||
SUPERVISOR_RESOLVED_HASH_PREFIX,
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
} from "./review-queue.js";
|
||||
|
||||
class FakeGarnet {
|
||||
readonly hashes = new Map<string, Record<string, string>>();
|
||||
readonly stream: StreamEntry[];
|
||||
|
||||
constructor(stream: StreamEntry[]) {
|
||||
this.stream = stream;
|
||||
}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
|
||||
const entries = this.stream.filter((entry) => {
|
||||
const afterStart = start === "-" || entry.id >= start;
|
||||
const beforeEnd = end === "+" || entry.id <= end;
|
||||
return afterStart && beforeEnd;
|
||||
});
|
||||
return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries);
|
||||
}
|
||||
|
||||
getHash(key: string): Promise<Record<string, string>> {
|
||||
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
|
||||
}
|
||||
|
||||
setHash(key: string, fields: Record<string, string>): Promise<void> {
|
||||
this.hashes.set(key, { ...fields });
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
expireKey(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
}
|
||||
|
||||
const packet = {
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "out-of-scope",
|
||||
sourceChain: ["onboarding", "moderation"],
|
||||
summary: "user asked about ads",
|
||||
userGoal: "광고 규칙 문의",
|
||||
};
|
||||
|
||||
const entries: StreamEntry[] = [
|
||||
{
|
||||
id: "1-0",
|
||||
message: {
|
||||
chatroomId: "room-1",
|
||||
packet: JSON.stringify(packet),
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "out-of-scope",
|
||||
requestId: "request-1",
|
||||
target: "moderation",
|
||||
type: "supervisor-handoff",
|
||||
userId: "user-1",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "2-0",
|
||||
message: {
|
||||
chatroomId: "room-2",
|
||||
queuedAt: "2026-05-18T01:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-2",
|
||||
target: "human-review",
|
||||
type: "supervisor-handoff",
|
||||
userId: "user-2",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe("supervisor review queue", () => {
|
||||
const firstEntry = entries[0]!;
|
||||
|
||||
it("parses a stream entry with the preserved handoff packet", () => {
|
||||
expect(parseSupervisorReviewEntry(firstEntry)).toMatchObject({
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
packet,
|
||||
reason: "out-of-scope",
|
||||
target: "moderation",
|
||||
});
|
||||
});
|
||||
|
||||
it("lists pending entries and filters resolved items by default", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
await resolveSupervisorReview(garnet, "1-0", {
|
||||
action: "manual-reply",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
|
||||
await expect(listSupervisorReview(garnet)).resolves.toHaveLength(1);
|
||||
await expect(
|
||||
listSupervisorReview(garnet, { includeResolved: true, reason: "out-of-scope" }),
|
||||
).resolves.toMatchObject([
|
||||
{
|
||||
id: "1-0",
|
||||
resolution: {
|
||||
action: "manual-reply",
|
||||
resolvedBy: "operator@example.com",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("gets a single entry with resolution metadata", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
||||
action: "dismiss",
|
||||
note: "duplicate",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
|
||||
await expect(getSupervisorReview(garnet, "1-0")).resolves.toMatchObject({
|
||||
id: "1-0",
|
||||
resolution: {
|
||||
action: "dismiss",
|
||||
note: "duplicate",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("computes reason totals and top source chains", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
||||
action: "reroute",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
|
||||
await expect(getSupervisorReviewStats(garnet)).resolves.toMatchObject({
|
||||
byReason: {
|
||||
"low-confidence": {
|
||||
pending: 1,
|
||||
resolved: 0,
|
||||
total: 1,
|
||||
},
|
||||
"out-of-scope": {
|
||||
pending: 0,
|
||||
resolved: 1,
|
||||
total: 1,
|
||||
},
|
||||
},
|
||||
pending: 1,
|
||||
resolved: 1,
|
||||
topSourceChains: [{ chain: "onboarding>moderation", count: 1 }],
|
||||
total: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,14 +1,344 @@
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorReviewQueueMaxLen,
|
||||
getSupervisorReviewResolutionTtlSec,
|
||||
} from "../../config.js";
|
||||
|
||||
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
|
||||
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
|
||||
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
|
||||
"lolympus-agent:review-resolved:";
|
||||
|
||||
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
|
||||
|
||||
export interface SupervisorReviewEntry {
|
||||
id: string;
|
||||
queuedAt: string;
|
||||
chatroomId: string;
|
||||
userId: string;
|
||||
requestId: string;
|
||||
reason: HandoffReason;
|
||||
target: string;
|
||||
packet?: HandoffPacket | undefined;
|
||||
extra?: Record<string, string> | undefined;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewResolution {
|
||||
resolvedAt: string;
|
||||
resolvedBy: string;
|
||||
action: SupervisorReviewAction;
|
||||
note?: string | undefined;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewItem extends SupervisorReviewEntry {
|
||||
resolution?: SupervisorReviewResolution | undefined;
|
||||
}
|
||||
|
||||
export interface ListSupervisorReviewOptions {
|
||||
includeResolved?: boolean | undefined;
|
||||
limit?: number | undefined;
|
||||
reason?: HandoffReason | undefined;
|
||||
since?: string | undefined;
|
||||
sinceQueuedAt?: string | undefined;
|
||||
until?: string | undefined;
|
||||
untilQueuedAt?: string | undefined;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewReasonStats {
|
||||
pending: number;
|
||||
resolved: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewStats {
|
||||
averagePendingAgeMs: number;
|
||||
byReason: Record<string, SupervisorReviewReasonStats>;
|
||||
pending: number;
|
||||
resolved: number;
|
||||
topSourceChains: Array<{ chain: string; count: number }>;
|
||||
total: number;
|
||||
}
|
||||
|
||||
type SupervisorReviewListGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"getHash" | "readStreamRange"
|
||||
>;
|
||||
|
||||
type SupervisorReviewResolveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"expireKey" | "setHash"
|
||||
>;
|
||||
|
||||
const BASE_FIELDS = new Set([
|
||||
"chatroomId",
|
||||
"packet",
|
||||
"queuedAt",
|
||||
"reason",
|
||||
"requestId",
|
||||
"target",
|
||||
"type",
|
||||
"userId",
|
||||
]);
|
||||
|
||||
function isHandoffReason(value: string | undefined): value is HandoffReason {
|
||||
return (
|
||||
value === "out-of-scope" ||
|
||||
value === "missing-tool" ||
|
||||
value === "low-confidence" ||
|
||||
value === "validation-failed" ||
|
||||
value === "same-agent-revisit" ||
|
||||
value === "max-hop" ||
|
||||
value === "high-risk-action" ||
|
||||
value === "policy-risk"
|
||||
);
|
||||
}
|
||||
|
||||
function isSupervisorReviewAction(
|
||||
value: string | undefined,
|
||||
): value is SupervisorReviewAction {
|
||||
return value === "manual-reply" || value === "reroute" || value === "dismiss";
|
||||
}
|
||||
|
||||
function parseOptionalPacket(value: string | undefined): HandoffPacket | undefined {
|
||||
if (!value?.trim() || value.trim() === "null") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
return typeof parsed === "object" && parsed !== null
|
||||
? (parsed as HandoffPacket)
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function parseResolution(
|
||||
fields: Record<string, string>,
|
||||
): SupervisorReviewResolution | undefined {
|
||||
if (
|
||||
!fields["resolvedAt"] ||
|
||||
!fields["resolvedBy"] ||
|
||||
!isSupervisorReviewAction(fields["action"])
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
action: fields["action"],
|
||||
resolvedAt: fields["resolvedAt"],
|
||||
resolvedBy: fields["resolvedBy"],
|
||||
...(fields["note"] ? { note: fields["note"] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolutionKey(id: string): string {
|
||||
return `${SUPERVISOR_RESOLVED_HASH_PREFIX}${id}`;
|
||||
}
|
||||
|
||||
function isAfterOrEqual(value: string, lowerBound: string | undefined): boolean {
|
||||
return !lowerBound || value >= lowerBound;
|
||||
}
|
||||
|
||||
function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean {
|
||||
return !upperBound || value <= upperBound;
|
||||
}
|
||||
|
||||
export async function appendSupervisorReview(
|
||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
await garnetClient.appendStream(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
{
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
},
|
||||
{ maxLen: getSupervisorReviewQueueMaxLen() },
|
||||
);
|
||||
}
|
||||
|
||||
export function parseSupervisorReviewEntry(
|
||||
streamEntry: StreamEntry,
|
||||
): SupervisorReviewEntry | null {
|
||||
const { message } = streamEntry;
|
||||
const reason = message["reason"];
|
||||
|
||||
if (
|
||||
!message["queuedAt"] ||
|
||||
!message["chatroomId"] ||
|
||||
!message["userId"] ||
|
||||
!message["requestId"] ||
|
||||
!message["target"] ||
|
||||
!isHandoffReason(reason)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const extra = Object.fromEntries(
|
||||
Object.entries(message).filter(([key]) => !BASE_FIELDS.has(key)),
|
||||
);
|
||||
const packet = parseOptionalPacket(message["packet"]);
|
||||
|
||||
return {
|
||||
chatroomId: message["chatroomId"],
|
||||
id: streamEntry.id,
|
||||
queuedAt: message["queuedAt"],
|
||||
reason,
|
||||
requestId: message["requestId"],
|
||||
target: message["target"],
|
||||
userId: message["userId"],
|
||||
...(packet ? { packet } : {}),
|
||||
...(Object.keys(extra).length > 0 ? { extra } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function getSupervisorReview(
|
||||
garnetClient: SupervisorReviewListGarnetClient,
|
||||
id: string,
|
||||
): Promise<SupervisorReviewItem | null> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
id,
|
||||
id,
|
||||
{ count: 1 },
|
||||
);
|
||||
const parsed = entries.flatMap((entry) => {
|
||||
const review = parseSupervisorReviewEntry(entry);
|
||||
return review ? [review] : [];
|
||||
})[0];
|
||||
|
||||
if (!parsed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const resolution = parseResolution(
|
||||
await garnetClient.getHash(resolutionKey(parsed.id)),
|
||||
);
|
||||
return {
|
||||
...parsed,
|
||||
...(resolution ? { resolution } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listSupervisorReview(
|
||||
garnetClient: SupervisorReviewListGarnetClient,
|
||||
options?: ListSupervisorReviewOptions,
|
||||
): Promise<SupervisorReviewItem[]> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
options?.since ?? "-",
|
||||
options?.until ?? "+",
|
||||
);
|
||||
const items: SupervisorReviewItem[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
const review = parseSupervisorReviewEntry(entry);
|
||||
if (!review) {
|
||||
continue;
|
||||
}
|
||||
if (options?.reason && review.reason !== options.reason) {
|
||||
continue;
|
||||
}
|
||||
if (!isAfterOrEqual(review.queuedAt, options?.sinceQueuedAt)) {
|
||||
continue;
|
||||
}
|
||||
if (!isBeforeOrEqual(review.queuedAt, options?.untilQueuedAt)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const resolution = parseResolution(
|
||||
await garnetClient.getHash(resolutionKey(review.id)),
|
||||
);
|
||||
if (resolution && !options?.includeResolved) {
|
||||
continue;
|
||||
}
|
||||
items.push({
|
||||
...review,
|
||||
...(resolution ? { resolution } : {}),
|
||||
});
|
||||
if (options?.limit && items.length >= options.limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
export async function resolveSupervisorReview(
|
||||
garnetClient: SupervisorReviewResolveGarnetClient,
|
||||
id: string,
|
||||
resolution: SupervisorReviewResolution,
|
||||
): Promise<void> {
|
||||
const fields: Record<string, string> = {
|
||||
action: resolution.action,
|
||||
resolvedAt: resolution.resolvedAt,
|
||||
resolvedBy: resolution.resolvedBy,
|
||||
};
|
||||
if (resolution.note?.trim()) {
|
||||
fields.note = resolution.note.trim();
|
||||
}
|
||||
|
||||
const key = resolutionKey(id);
|
||||
await garnetClient.setHash(key, fields);
|
||||
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
|
||||
}
|
||||
|
||||
export async function getSupervisorReviewStats(
|
||||
garnetClient: SupervisorReviewListGarnetClient,
|
||||
options?: Omit<ListSupervisorReviewOptions, "includeResolved">,
|
||||
): Promise<SupervisorReviewStats> {
|
||||
const nowMs = Date.now();
|
||||
const items = await listSupervisorReview(garnetClient, {
|
||||
...options,
|
||||
includeResolved: true,
|
||||
});
|
||||
const byReason: Record<string, SupervisorReviewReasonStats> = {};
|
||||
const sourceChains = new Map<string, number>();
|
||||
let pendingAgeMs = 0;
|
||||
let pending = 0;
|
||||
let resolved = 0;
|
||||
|
||||
for (const item of items) {
|
||||
const reasonStats = byReason[item.reason] ?? {
|
||||
pending: 0,
|
||||
resolved: 0,
|
||||
total: 0,
|
||||
};
|
||||
reasonStats.total += 1;
|
||||
|
||||
if (item.resolution) {
|
||||
resolved += 1;
|
||||
reasonStats.resolved += 1;
|
||||
} else {
|
||||
pending += 1;
|
||||
reasonStats.pending += 1;
|
||||
const queuedAtMs = Date.parse(item.queuedAt);
|
||||
if (Number.isFinite(queuedAtMs)) {
|
||||
pendingAgeMs += Math.max(0, nowMs - queuedAtMs);
|
||||
}
|
||||
}
|
||||
|
||||
byReason[item.reason] = reasonStats;
|
||||
|
||||
const chain = item.packet?.sourceChain?.join(">");
|
||||
if (chain) {
|
||||
sourceChains.set(chain, (sourceChains.get(chain) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
averagePendingAgeMs: pending === 0 ? 0 : Math.round(pendingAgeMs / pending),
|
||||
byReason,
|
||||
pending,
|
||||
resolved,
|
||||
topSourceChains: [...sourceChains.entries()]
|
||||
.map(([chain, count]) => ({ chain, count }))
|
||||
.sort((left, right) => right.count - left.count)
|
||||
.slice(0, 5),
|
||||
total: items.length,
|
||||
};
|
||||
}
|
||||
@@ -82,6 +82,15 @@ describe("SupervisorSkill", () => {
|
||||
type: "supervisor-handoff",
|
||||
},
|
||||
});
|
||||
expect(garnet.streams[0]?.fields["packet"]).toEqual(expect.any(String));
|
||||
const queuedPacket: unknown = JSON.parse(
|
||||
String(garnet.streams[0]?.fields["packet"]),
|
||||
);
|
||||
expect(queuedPacket).toMatchObject({
|
||||
handoffReason: "max-hop",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
userGoal: "가입 상태 확인",
|
||||
});
|
||||
expect(nats.published[0]).toMatchObject({
|
||||
kind: "supervisor-handoff",
|
||||
metric: "agent.supervisor_handoff",
|
||||
|
||||
@@ -56,6 +56,7 @@ export class SupervisorSkill implements Skill {
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
chatroomId: request.chatroomId,
|
||||
packet: JSON.stringify(packet ?? null),
|
||||
reason,
|
||||
requestId: request.requestId,
|
||||
target: target ?? "human-review",
|
||||
|
||||
Reference in new issue
Block a user