Close supervisor review loop and add handoff eval

This commit is contained in:
dahyeon committed 2026-05-18 18:04:01 -07:00
1 parent c14adac162
commit f0b00e5fd2
20 files changed
+2017 -129

No files matched your search

+16
View File
@@ -151,6 +151,10 @@ describe("Agent handoff dispatch", () => {
expect(nats.responses).toHaveLength(1);
expect(nats.responses[0]?.text).toBe("final answer");
expect(nats.responses[0]?.metadata).toMatchObject({
handoffChain: ["source", "target"],
handoffDepth: 1,
});
expect(store.messages.map((message) => message.content)).toEqual([
"final answer",
]);
@@ -189,6 +193,10 @@ describe("Agent handoff dispatch", () => {
expect(nats.responses[0]?.text).toBe("handled elsewhere");
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
expect(nats.responses[0]?.metadata).toMatchObject({
handoffChain: ["source", "target"],
handoffDepth: 1,
});
});
it("routes runtime verifier escalations to supervisor", async () => {
@@ -228,6 +236,10 @@ describe("Agent handoff dispatch", () => {
expect(nats.responses[0]?.text).toBe("sent to review");
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
expect(nats.responses[0]?.metadata).toMatchObject({
handoffChain: ["moderation", "supervisor"],
handoffDepth: 1,
});
} finally {
if (previous === undefined) {
delete process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
@@ -297,5 +309,9 @@ describe("Agent handoff dispatch", () => {
expect(nats.responses[0]?.text).toBe("supervisor review");
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
expect(nats.responses[0]?.metadata).toMatchObject({
handoffChain: ["source", "supervisor"],
handoffDepth: 1,
});
});
});
+24 -4
View File
@@ -282,10 +282,11 @@ export class Agent {
response: verified,
skill,
});
const annotated = this.withHandoffMetadata(escalated, chain, depth);
const handoff = readHandoffRequest(escalated);
const handoff = readHandoffRequest(annotated);
if (!handoff) {
return escalated;
return annotated;
}
const next = this.prepareHandoff({
@@ -296,11 +297,15 @@ export class Agent {
handoff,
history,
request,
response: escalated,
response: annotated,
});
if (!next) {
return this.makeSupervisorFallback(request, handoff.reason);
return this.withHandoffMetadata(
this.makeSupervisorFallback(request, handoff.reason),
[...chain, "supervisor"],
depth + 1,
);
}
return this.dispatch(next);
@@ -308,6 +313,21 @@ export class Agent {
);
}
private withHandoffMetadata(
response: AgentResponse,
chain: string[],
depth: number,
): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffChain: chain,
handoffDepth: depth,
},
};
}
private async runSkillOnce(input: {
chain: string[];
depth: number;
+13
View File
@@ -0,0 +1,13 @@
import { GarnetClient } from "./garnet.js";
export async function connectGarnetForCli(): Promise<{
close: () => Promise<void>;
garnetClient: GarnetClient;
}> {
const garnetClient = new GarnetClient();
await garnetClient.connect();
return {
close: () => garnetClient.close(),
garnetClient,
};
}
+11
View File
@@ -218,6 +218,17 @@ export function getEvalReviewDedupeTtlSec(): number {
return getPositiveNumber("EVAL_REVIEW_DEDUPE_TTL_SEC", 604_800);
}
export function getSupervisorReviewQueueMaxLen(): number {
return getPositiveNumber("SUPERVISOR_REVIEW_QUEUE_MAXLEN", 10_000);
}
export function getSupervisorReviewResolutionTtlSec(): number {
return getPositiveNumber(
"SUPERVISOR_REVIEW_RESOLUTION_TTL_SEC",
90 * 24 * 60 * 60,
);
}
export function getEvalCalibrationMinCorr(): number {
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
}
+113
View File
@@ -0,0 +1,113 @@
export interface ParsedArgs {
command: string;
flags: Map<string, string | true>;
positionals: string[];
}
export function parsePositiveInteger(
value: string | undefined,
): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
export function parsePositiveNumber(
value: string | undefined,
): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
}
export function parseDurationMs(value: string | undefined): number {
const raw = value?.trim().toLowerCase() || "7d";
const match = raw.match(/^(\d+)(h|d)$/u);
if (!match) {
throw new Error("--since must use a duration like 24h, 7d, or 30d");
}
const amount = Number(match[1]);
const unit = match[2];
if (!Number.isInteger(amount) || amount <= 0) {
throw new Error("--since must be a positive duration");
}
return unit === "h"
? amount * 60 * 60 * 1000
: amount * 24 * 60 * 60 * 1000;
}
export function parseCsv(value: string | undefined): string[] {
if (!value?.trim()) {
return [];
}
return value
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
export function parseArgs(argv: string[]): ParsedArgs {
const [command = "run", ...rest] = argv;
const flags = new Map<string, string | true>();
const positionals: string[] = [];
for (let index = 0; index < rest.length; index++) {
const token = rest[index];
if (!token) {
continue;
}
if (!token.startsWith("--")) {
positionals.push(token);
continue;
}
const withoutPrefix = token.slice(2);
const equalsIndex = withoutPrefix.indexOf("=");
if (equalsIndex >= 0) {
const key = withoutPrefix.slice(0, equalsIndex);
const value = withoutPrefix.slice(equalsIndex + 1);
flags.set(key, value || true);
continue;
}
const next = rest[index + 1];
if (next && !next.startsWith("--")) {
flags.set(withoutPrefix, next);
index += 1;
continue;
}
flags.set(withoutPrefix, true);
}
return {
command,
flags,
positionals,
};
}
export function getFlagString(
flags: Map<string, string | true>,
key: string,
): string | undefined {
const value = flags.get(key);
return typeof value === "string" ? value : undefined;
}
export function hasFlag(
flags: Map<string, string | true>,
key: string,
): boolean {
return flags.has(key);
}
+9 -108
View File
@@ -1,51 +1,15 @@
import type { MongoConnectionLike } from "./archive.js";
import type { SliceKey, SliceReportRow } from "./aggregator.js";
import type { GarnetClient } from "@common/garnet";
function parsePositiveInteger(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
function parsePositiveNumber(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
}
function parseDurationMs(value: string | undefined): number {
const raw = value?.trim().toLowerCase() || "7d";
const match = raw.match(/^(\d+)(h|d)$/u);
if (!match) {
throw new Error("--since must use a duration like 24h, 7d, or 30d");
}
const amount = Number(match[1]);
const unit = match[2];
if (!Number.isInteger(amount) || amount <= 0) {
throw new Error("--since must be a positive duration");
}
return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000;
}
function parseCsv(value: string | undefined): string[] {
if (!value?.trim()) {
return [];
}
return value
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
import {
getFlagString,
hasFlag,
parseArgs,
parseCsv,
parseDurationMs,
parsePositiveInteger,
parsePositiveNumber,
} from "./cli-args.js";
function parseSlice(value: string | undefined): SliceKey {
if (!value?.trim()) {
@@ -85,69 +49,6 @@ function parseSlice(value: string | undefined): SliceKey {
};
}
interface ParsedArgs {
command: string;
flags: Map<string, string | true>;
positionals: string[];
}
function parseArgs(argv: string[]): ParsedArgs {
const [command = "run", ...rest] = argv;
const flags = new Map<string, string | true>();
const positionals: string[] = [];
for (let index = 0; index < rest.length; index++) {
const token = rest[index];
if (!token) {
continue;
}
if (!token.startsWith("--")) {
positionals.push(token);
continue;
}
const withoutPrefix = token.slice(2);
const equalsIndex = withoutPrefix.indexOf("=");
if (equalsIndex >= 0) {
const key = withoutPrefix.slice(0, equalsIndex);
const value = withoutPrefix.slice(equalsIndex + 1);
flags.set(key, value || true);
continue;
}
const next = rest[index + 1];
if (next && !next.startsWith("--")) {
flags.set(withoutPrefix, next);
index += 1;
continue;
}
flags.set(withoutPrefix, true);
}
return {
command,
flags,
positionals,
};
}
function getFlagString(
flags: Map<string, string | true>,
key: string,
): string | undefined {
const value = flags.get(key);
return typeof value === "string" ? value : undefined;
}
function hasFlag(
flags: Map<string, string | true>,
key: string,
): boolean {
return flags.has(key);
}
function printRunSummary(report: {
generatedAt: string;
summary: {
+6
View File
@@ -0,0 +1,6 @@
{"id":"case_handoff_onboarding_to_moderation_ad","name":"onboarding routes community rule question to moderation","input":{"request":{"requestId":"handoff-1","skill":"onboarding","chatroomId":"room-handoff","userId":"user-1","nickname":"tester","trigger":"user-message","userMessage":"광고 올려도 돼요?","context":{"evalHandoffPlan":[{"from":"onboarding","to":"moderation","reason":"out-of-scope","taskSpec":"광고/홍보 규칙 문의 처리"}]}},"conversationHistory":[{"role":"user","content":"광고 올려도 돼요?","timestamp":"2026-05-18T00:00:00.000Z"}]},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"out-of-scope","expectedTarget":"moderation"},"expectedFinalSkill":"moderation","expectedSourceChain":{"mode":"contains","chain":["moderation"]}}}
{"id":"case_handoff_moderation_to_onboarding_welcome","name":"moderation routes onboarding question to onboarding","input":{"request":{"requestId":"handoff-2","skill":"moderation","chatroomId":"room-handoff","userId":"user-2","nickname":"tester","trigger":"user-message","userMessage":"신규 가입 안내랑 닉네임 규칙 알려줘","context":{"evalHandoffPlan":[{"from":"moderation","to":"onboarding","reason":"out-of-scope","taskSpec":"신규 가입 안내 처리"}]}},"conversationHistory":[{"role":"user","content":"신규 가입 안내랑 닉네임 규칙 알려줘","timestamp":"2026-05-18T00:00:00.000Z"}]},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"out-of-scope","expectedTarget":"onboarding"},"expectedFinalSkill":"onboarding","expectedSourceChain":{"mode":"exact","chain":["moderation","onboarding"]}}}
{"id":"case_handoff_low_confidence_to_supervisor","name":"unclear handoff escalates to supervisor","input":{"request":{"requestId":"handoff-3","skill":"onboarding","chatroomId":"room-handoff","userId":"user-3","nickname":"tester","trigger":"user-message","userMessage":"음... 이거랑 저거 그거 어떻게 해?","context":{"evalHandoffPlan":[{"from":"onboarding","to":"supervisor","reason":"low-confidence","taskSpec":"의도 불명확 케이스 운영자 검토"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"low-confidence","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["onboarding","supervisor"]}}}
{"id":"case_handoff_validation_retry_then_supervisor","name":"validation failure escalates to supervisor","input":{"request":{"requestId":"handoff-4","skill":"moderation","chatroomId":"room-handoff","userId":"user-4","nickname":"tester","trigger":"user-message","userMessage":"규칙 답변에 링크를 넣어도 돼?","context":{"evalHandoffPlan":[{"from":"moderation","to":"supervisor","reason":"validation-failed","taskSpec":"runtime verifier failed after retry"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"validation-failed","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["moderation","supervisor"]}}}
{"id":"case_handoff_aba_loop_blocked","name":"aba loop is rerouted to supervisor","input":{"request":{"requestId":"handoff-5","skill":"onboarding","chatroomId":"room-handoff","userId":"user-5","nickname":"tester","trigger":"user-message","userMessage":"가입인지 규칙인지 헷갈리는 질문","context":{"evalBudgetMaxSameAgentVisits":1,"evalHandoffPlan":[{"from":"onboarding","to":"moderation","reason":"out-of-scope","taskSpec":"규칙 확인"},{"from":"moderation","to":"onboarding","reason":"out-of-scope","taskSpec":"가입 상태 확인"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"same-agent-revisit","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["onboarding","moderation","supervisor"]}}}
{"id":"case_handoff_high_risk_to_supervisor","name":"high risk action escalates to supervisor","input":{"request":{"requestId":"handoff-6","skill":"moderation","chatroomId":"room-handoff","userId":"user-6","nickname":"tester","trigger":"user-message","userMessage":"이 유저 바로 밴해줘","context":{"evalHandoffPlan":[{"from":"moderation","to":"supervisor","reason":"high-risk-action","risk":"high","taskSpec":"제재성 고위험 요청 운영자 검토"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"high-risk-action","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["moderation","supervisor"]}}}
+412
View File
@@ -0,0 +1,412 @@
import { BudgetTracker, type BudgetVerdict } from "@agent/budget";
import { buildHandoffPacket } from "@agent/handoff-packet";
import { selectSkillForRequest } from "@agent/agent";
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
HandoffReason,
HandoffRequest,
} from "@agent/types";
import { parseControlTokens } from "@skills/control-token-parser";
import { decide as decideEscalation } from "@skills/escalation/trigger";
import type { Skill } from "@skills/skill";
export interface EvalDispatchResult {
chain: string[];
finalSkill: string;
response: AgentResponse;
}
interface EvalHandoffPlanStep {
confidence?: number | undefined;
expectedOutput?: string | undefined;
from: string;
reason: HandoffReason;
risk?: HandoffRequest["risk"] | undefined;
taskSpec?: string | undefined;
to: string;
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
const rawPlan = asRecord(request.context)["evalHandoffPlan"];
if (!Array.isArray(rawPlan)) {
return [];
}
return rawPlan.flatMap((item): EvalHandoffPlanStep[] => {
const record = asRecord(item);
const from = record["from"];
const to = record["to"];
const reason = record["reason"];
if (
typeof from !== "string" ||
typeof to !== "string" ||
!isHandoffReason(reason)
) {
return [];
}
return [
{
confidence:
typeof record["confidence"] === "number"
? record["confidence"]
: undefined,
expectedOutput:
typeof record["expectedOutput"] === "string"
? record["expectedOutput"]
: undefined,
from,
reason,
risk:
record["risk"] === "low" ||
record["risk"] === "medium" ||
record["risk"] === "high"
? record["risk"]
: undefined,
taskSpec:
typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
to,
},
];
});
}
function isHandoffReason(value: unknown): value is HandoffReason {
return (
value === "out-of-scope" ||
value === "missing-tool" ||
value === "low-confidence" ||
value === "validation-failed" ||
value === "same-agent-revisit" ||
value === "max-hop" ||
value === "high-risk-action" ||
value === "policy-risk"
);
}
function readPositiveNumber(
context: Record<string, unknown> | undefined,
key: string,
fallback: number,
): number {
const value = context?.[key];
return typeof value === "number" && Number.isFinite(value) && value > 0
? value
: fallback;
}
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
if (verdict === "same-agent-loop") {
return "same-agent-revisit";
}
if (verdict === "max-hops") {
return "max-hop";
}
return "policy-risk";
}
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
const value = response.metadata?.["handoffRequest"];
if (typeof value !== "object" || value === null) {
return null;
}
const request = value as Partial<HandoffRequest>;
if (
typeof request.targetAgent !== "string" ||
typeof request.reason !== "string" ||
typeof request.taskSpec !== "string"
) {
return null;
}
return {
confidence:
typeof request.confidence === "number" ? request.confidence : response.confidence,
evidenceRefs: Array.isArray(request.evidenceRefs)
? request.evidenceRefs.filter((item): item is string => typeof item === "string")
: [],
expectedOutput:
typeof request.expectedOutput === "string"
? request.expectedOutput
: "text",
reason: request.reason,
risk:
request.risk === "low" || request.risk === "medium" || request.risk === "high"
? request.risk
: "medium",
targetAgent: request.targetAgent,
taskSpec: request.taskSpec,
...(request.userVisibleMessage
? { userVisibleMessage: request.userVisibleMessage }
: {}),
};
}
function normalizeControlTokens(
response: AgentResponse,
currentSkill: string,
): AgentResponse {
const parsed = parseControlTokens(response.text);
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
if (handoff?.kind !== "handoff") {
return {
...response,
text: parsed.stripped,
};
}
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: handoff.reason,
risk: "medium",
targetAgent: handoff.targetAgent,
taskSpec: `Continue handling from ${currentSkill}`,
} satisfies HandoffRequest,
},
text: parsed.stripped,
};
}
function withHandoffMetadata(
response: AgentResponse,
chain: string[],
depth: number,
lastHandoff: HandoffRequest | undefined,
): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffChain: chain,
handoffDepth: depth,
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
},
};
}
function plannedHandoffResponse(
request: AgentRequest,
step: EvalHandoffPlanStep,
): AgentResponse {
return {
confidence: step.confidence ?? 0.72,
metadata: {
handoffRequest: {
confidence: step.confidence ?? 0.72,
evidenceRefs: [],
expectedOutput: step.expectedOutput ?? "text",
reason: step.reason,
risk: step.risk ?? "medium",
targetAgent: step.to,
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
} satisfies HandoffRequest,
},
requestId: request.requestId,
source: "template",
text: `handoff ${step.from} to ${step.to}`,
};
}
async function runActualSkill(
request: AgentRequest,
history: ConversationMessage[],
skill: Skill,
budget: BudgetTracker,
): Promise<AgentResponse> {
const response = normalizeControlTokens(
await skill.handle(request, history),
skill.name,
);
if (!response.text.trim() || readHandoffRequest(response)) {
return response;
}
const decision = decideEscalation({
budget: budget.snapshot(),
capabilityCard: skill.capabilityCard?.(),
classified: { confidence: response.confidence },
targetAgent: "supervisor",
taskSpec: `Review ${skill.name} response for ${request.trigger}.`,
});
if (!decision.handoff) {
return response;
}
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: decision.handoff,
},
};
}
export async function runEvalDispatch(input: {
history?: ConversationMessage[] | undefined;
request: AgentRequest;
skills: readonly Skill[];
}): Promise<EvalDispatchResult> {
const context = input.request.context;
const plan = readPlan(input.request);
const useActualSkills = asRecord(context)["evalUseActualSkills"] === true;
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5),
maxHandoffs: readPositiveNumber(context, "evalBudgetMaxHandoffs", 4),
maxSameAgentVisits: readPositiveNumber(
context,
"evalBudgetMaxSameAgentVisits",
2,
),
maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20),
});
let request = input.request;
let skill = selectSkillForRequest(input.skills, request);
if (!skill) {
throw new Error(`No skill can handle eval request: ${request.skill}`);
}
let chain = [skill.name];
let lastHandoff: HandoffRequest | undefined;
let planIndex = 0;
budget.recordVisit(skill.name);
for (let depth = 0; depth <= 8; depth++) {
const planned = plan[planIndex]?.from === skill.name ? plan[planIndex] : undefined;
const response = planned
? plannedHandoffResponse(request, planned)
: useActualSkills
? await runActualSkill(request, input.history ?? [], skill, budget)
: {
confidence: 1,
requestId: request.requestId,
source: "template" as const,
text: `handoff eval final: ${skill.name}`,
};
if (planned) {
planIndex += 1;
}
const handoff = readHandoffRequest(response);
if (!handoff) {
const finalResponse = withHandoffMetadata(
response,
chain,
chain.length - 1,
lastHandoff,
);
return {
chain,
finalSkill: skill.name,
response: finalResponse,
};
}
const verdict = budget.assertBeforeHandoff(handoff.targetAgent);
const resolved: HandoffRequest =
verdict === "ok"
? handoff
: {
...handoff,
reason: budgetVerdictToReason(verdict),
targetAgent: "supervisor",
};
const nextSkill = selectSkillForRequest(input.skills, {
...request,
skill: resolved.targetAgent,
trigger: "handoff",
});
if (!nextSkill) {
const fallbackChain = [...chain, "supervisor"];
const fallbackHandoff = {
...resolved,
targetAgent: "supervisor",
};
return {
chain: fallbackChain,
finalSkill: "supervisor",
response: withHandoffMetadata(
{
confidence: 0.4,
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
},
fallbackChain,
fallbackChain.length - 1,
fallbackHandoff,
),
};
}
budget.recordHandoff(nextSkill.name);
const nextChain = [...chain, nextSkill.name];
lastHandoff = {
...resolved,
targetAgent: nextSkill.name,
};
const packet = buildHandoffPacket({
handoff: lastHandoff,
history: input.history,
request,
response,
sourceChain: nextChain,
state: {
source: skill.name,
},
});
request = {
...request,
context: {
...(request.context ?? {}),
handoffPacket: packet,
},
skill: nextSkill.name,
trigger: "handoff",
};
skill = nextSkill;
chain = nextChain;
}
const finalHandoff: HandoffRequest = {
confidence: 0.4,
evidenceRefs: [],
expectedOutput: "text",
reason: "max-hop",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "Eval dispatch exceeded max loop depth.",
};
const fallbackChain = chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"];
return {
chain: fallbackChain,
finalSkill: "supervisor",
response: withHandoffMetadata(
{
confidence: 0.4,
requestId: input.request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
},
fallbackChain,
fallbackChain.length - 1,
finalHandoff,
),
};
}
+102
View File
@@ -0,0 +1,102 @@
import { describe, expect, it } from "vitest";
import type { AgentResponse } from "@agent/types";
import { judgeHandoff } from "./handoff-judge.js";
const response: AgentResponse = {
confidence: 0.8,
metadata: {
handoffChain: ["onboarding", "moderation", "supervisor"],
handoffLastRequest: {
confidence: 0.6,
evidenceRefs: [],
expectedOutput: "text",
reason: "same-agent-revisit",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "loop blocked",
},
},
requestId: "request-1",
source: "template",
text: "sent to review",
};
describe("judgeHandoff", () => {
it("passes exact chain, final skill, target, and reason checks", () => {
expect(
judgeHandoff(
{
expectedFinalSkill: "supervisor",
expectedHandoff: {
expectedReason: "same-agent-revisit",
expectedTarget: "supervisor",
shouldHandoff: true,
},
expectedSourceChain: {
chain: ["onboarding", "moderation", "supervisor"],
mode: "exact",
},
},
response,
),
).toMatchObject({
actualChain: ["onboarding", "moderation", "supervisor"],
actualFinalSkill: "supervisor",
actualHandoffReason: "same-agent-revisit",
actualHandoffTarget: "supervisor",
passed: true,
});
});
it("supports prefix chain checks", () => {
expect(
judgeHandoff(
{
expectedSourceChain: {
chain: ["onboarding", "moderation"],
mode: "prefix",
},
},
response,
).passed,
).toBe(true);
});
it("supports contains chain checks", () => {
expect(
judgeHandoff(
{
expectedSourceChain: {
chain: ["moderation"],
mode: "contains",
},
},
response,
).passed,
).toBe(true);
});
it("fails when required metadata is missing", () => {
const result = judgeHandoff(
{
expectedFinalSkill: "supervisor",
expectedHandoff: { shouldHandoff: true },
expectedSourceChain: {
chain: ["supervisor"],
mode: "contains",
},
},
{ ...response, metadata: {} },
);
expect(result.passed).toBe(false);
expect(result.checks.filter((check) => !check.passed)).toHaveLength(3);
});
it("skips unspecified checks", () => {
const result = judgeHandoff({}, response);
expect(result.passed).toBe(true);
expect(result.checks.every((check) => check.skipped)).toBe(true);
});
});
+152
View File
@@ -0,0 +1,152 @@
import type { AgentResponse, HandoffRequest } from "@agent/types";
import type {
EvalCaseExpected,
HandoffJudgeCheck,
HandoffJudgeResult,
} from "../types.js";
function asStringArray(value: unknown): string[] {
if (!Array.isArray(value)) {
return [];
}
return value.filter((item): item is string => typeof item === "string");
}
function readHandoffRequest(
value: unknown,
): Pick<HandoffRequest, "reason" | "targetAgent"> | null {
if (typeof value !== "object" || value === null) {
return null;
}
const record = value as Partial<HandoffRequest>;
if (typeof record.reason !== "string" || typeof record.targetAgent !== "string") {
return null;
}
return {
reason: record.reason,
targetAgent: record.targetAgent,
};
}
function chainMatches(
actual: readonly string[],
expected: readonly string[],
mode: "contains" | "exact" | "prefix",
): boolean {
if (mode === "exact") {
return (
actual.length === expected.length &&
expected.every((skill, index) => actual[index] === skill)
);
}
if (mode === "prefix") {
return expected.every((skill, index) => actual[index] === skill);
}
if (expected.length === 0) {
return true;
}
return actual.some((_skill, startIndex) =>
expected.every((skill, offset) => actual[startIndex + offset] === skill),
);
}
function skippedCheck(name: HandoffJudgeCheck["name"]): HandoffJudgeCheck {
return {
name,
passed: true,
skipped: true,
};
}
export function judgeHandoff(
expected: EvalCaseExpected,
response: AgentResponse,
finalSkill?: string,
): HandoffJudgeResult {
const metadata = response.metadata ?? {};
const actualChain = asStringArray(metadata["handoffChain"]);
const finalSkillFromChain = actualChain.at(-1);
const actualFinalSkill =
finalSkill ?? (typeof finalSkillFromChain === "string" ? finalSkillFromChain : "");
const handoff =
readHandoffRequest(metadata["handoffRequest"]) ??
readHandoffRequest(metadata["handoffLastRequest"]);
const actualHandoffReason = handoff?.reason;
const actualHandoffTarget = handoff?.targetAgent;
const checks: HandoffJudgeCheck[] = [];
if (expected.expectedHandoff?.shouldHandoff === undefined) {
checks.push(skippedCheck("should-handoff"));
} else {
const didHandoff = actualChain.length > 1 || Boolean(handoff);
const passed = didHandoff === expected.expectedHandoff.shouldHandoff;
checks.push({
detail: `expected=${expected.expectedHandoff.shouldHandoff} actual=${didHandoff}`,
name: "should-handoff",
passed,
skipped: false,
});
}
if (!expected.expectedHandoff?.expectedReason) {
checks.push(skippedCheck("handoff-reason"));
} else {
const passed = actualHandoffReason === expected.expectedHandoff.expectedReason;
checks.push({
detail: `expected=${expected.expectedHandoff.expectedReason} actual=${actualHandoffReason ?? "-"}`,
name: "handoff-reason",
passed,
skipped: false,
});
}
if (!expected.expectedHandoff?.expectedTarget) {
checks.push(skippedCheck("handoff-target"));
} else {
const passed = actualHandoffTarget === expected.expectedHandoff.expectedTarget;
checks.push({
detail: `expected=${expected.expectedHandoff.expectedTarget} actual=${actualHandoffTarget ?? "-"}`,
name: "handoff-target",
passed,
skipped: false,
});
}
if (!expected.expectedFinalSkill) {
checks.push(skippedCheck("final-skill"));
} else {
const passed = actualFinalSkill === expected.expectedFinalSkill;
checks.push({
detail: `expected=${expected.expectedFinalSkill} actual=${actualFinalSkill || "-"}`,
name: "final-skill",
passed,
skipped: false,
});
}
if (!expected.expectedSourceChain) {
checks.push(skippedCheck("chain"));
} else {
const passed = chainMatches(
actualChain,
expected.expectedSourceChain.chain,
expected.expectedSourceChain.mode,
);
checks.push({
detail: `mode=${expected.expectedSourceChain.mode} expected=${expected.expectedSourceChain.chain.join(">")} actual=${actualChain.join(">") || "-"}`,
name: "chain",
passed,
skipped: false,
});
}
return {
actualChain,
actualFinalSkill,
...(actualHandoffReason ? { actualHandoffReason } : {}),
...(actualHandoffTarget ? { actualHandoffTarget } : {}),
checks,
passed: checks.every((check) => check.skipped || check.passed),
};
}
+1 -1
View File
@@ -25,7 +25,7 @@ export interface ReviewCliOptions {
resetPending?: boolean | undefined;
}
function getReviewerId(): string {
export function getReviewerId(): string {
const configured = process.env["REVIEWER_ID"]?.trim();
if (configured) {
return configured;
+231 -10
View File
@@ -2,8 +2,8 @@ import { existsSync } from "node:fs";
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import { basename, dirname, resolve } from "node:path";
import type { AgentRequest } from "@agent/types";
import { GarnetClient } from "@common/garnet";
import type { AgentRequest, AgentResponse, HandoffReason } from "@agent/types";
import { GarnetClient, type StreamEntry } from "@common/garnet";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { NatsClient } from "@common/nats";
import { getConversationSummaryKey } from "@conv/conversation-store";
@@ -17,6 +17,7 @@ import type { Skill } from "@skills/skill";
import { ModerationSkill } from "@skills/moderation/index";
import { classifyModerationIntent } from "@skills/moderation/intents";
import { OnboardingSkill } from "@skills/onboarding/index";
import { SupervisorSkill } from "@skills/supervisor/index";
import {
ProcessTraceCollector,
runWithProcessTrace,
@@ -30,6 +31,8 @@ import type {
} from "@skills/onboarding/state/onboarding-state";
import { noopShadowEvalPublisher } from "./shadow-bus.js";
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
import { runEvalDispatch } from "./handoff-dispatch.js";
import { judgeHandoff } from "./judges/handoff-judge.js";
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
import { judgeProcessRule } from "./judges/process-judge.js";
import { judgeRuleCase } from "./judges/rule-judge.js";
@@ -72,11 +75,17 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
deps.room,
noopShadowEvalPublisher,
),
supervisor: (deps) =>
new SupervisorSkill(deps.garnet, {
publish: <T = unknown>(_subject: string, _payload: T): Promise<void> =>
Promise.resolve(),
}),
};
class InMemoryGarnetClient extends GarnetClient {
private readonly hashStore = new Map<string, Record<string, string>>();
private readonly stringStore = new Map<string, string>();
private readonly streamStore = new Map<string, StreamEntry[]>();
getJson<T>(key: string): Promise<T | null> {
const raw = this.stringStore.get(key);
@@ -105,6 +114,18 @@ class InMemoryGarnetClient extends GarnetClient {
return Promise.resolve();
}
setStringIfAbsent(
key: string,
value: string,
_ttlSeconds?: number,
): Promise<boolean> {
if (this.stringStore.has(key)) {
return Promise.resolve(false);
}
this.stringStore.set(key, value);
return Promise.resolve(true);
}
delete(key: string): Promise<number> {
const stringDeleted = this.stringStore.delete(key);
const hashDeleted = this.hashStore.delete(key);
@@ -122,6 +143,39 @@ class InMemoryGarnetClient extends GarnetClient {
this.hashStore.set(key, { ...fields });
return Promise.resolve();
}
expireKey(_key: string, _ttlSeconds: number): Promise<boolean> {
return Promise.resolve(true);
}
appendStream(
key: string,
fields: Record<string, string>,
_options?: { maxLen?: number | undefined },
): Promise<string> {
const entries = this.streamStore.get(key) ?? [];
const id = `${entries.length + 1}-0`;
entries.push({
id,
message: { ...fields },
});
this.streamStore.set(key, entries);
return Promise.resolve(id);
}
readStreamRange(
key: string,
start: string,
end: string,
options?: { count?: number | undefined },
): Promise<StreamEntry[]> {
const entries = (this.streamStore.get(key) ?? []).filter((entry) => {
const afterStart = start === "-" || entry.id >= start;
const beforeEnd = end === "+" || entry.id <= end;
return afterStart && beforeEnd;
});
return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries);
}
}
class InMemoryRoomContextService extends RoomContextService {
@@ -277,6 +331,19 @@ function getStringArrayField(
return strings.length > 0 ? strings : [];
}
function isHandoffReason(value: string | undefined): value is HandoffReason {
return (
value === "out-of-scope" ||
value === "missing-tool" ||
value === "low-confidence" ||
value === "validation-failed" ||
value === "same-agent-revisit" ||
value === "max-hop" ||
value === "high-risk-action" ||
value === "policy-risk"
);
}
function toAgentRequest(raw: Record<string, unknown>): AgentRequest {
const requestId = getStringField(raw, "requestId");
const skill = getStringField(raw, "skill");
@@ -360,10 +427,14 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
const responseMode = getStringField(raw, "responseMode");
const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny");
const mustNotInclude = getStringArrayField(raw, "mustNotInclude");
const expectedFinalSkill = getStringField(raw, "expectedFinalSkill");
if (intent) {
expected.intent = intent;
}
if (expectedFinalSkill) {
expected.expectedFinalSkill = expectedFinalSkill;
}
if (responseMode) {
expected.responseMode = responseMode;
}
@@ -409,6 +480,42 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
}
}
const expectedHandoffRaw = raw["expectedHandoff"];
if (isRecord(expectedHandoffRaw)) {
const expectedReason = getStringField(expectedHandoffRaw, "expectedReason");
const expectedTarget = getStringField(expectedHandoffRaw, "expectedTarget");
const shouldHandoff = expectedHandoffRaw["shouldHandoff"];
const expectedHandoff: NonNullable<EvalCaseExpected["expectedHandoff"]> = {};
if (typeof shouldHandoff === "boolean") {
expectedHandoff.shouldHandoff = shouldHandoff;
}
if (isHandoffReason(expectedReason)) {
expectedHandoff.expectedReason = expectedReason;
}
if (expectedTarget) {
expectedHandoff.expectedTarget = expectedTarget;
}
if (Object.keys(expectedHandoff).length > 0) {
expected.expectedHandoff = expectedHandoff;
}
}
const expectedSourceChainRaw = raw["expectedSourceChain"];
if (isRecord(expectedSourceChainRaw)) {
const mode = getStringField(expectedSourceChainRaw, "mode");
const chain = getStringArrayField(expectedSourceChainRaw, "chain");
if (
chain &&
(mode === "exact" || mode === "prefix" || mode === "contains")
) {
expected.expectedSourceChain = {
chain,
mode,
};
}
}
return expected;
}
@@ -538,6 +645,7 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary {
const llmResults = results.flatMap((result) =>
result.llmJudge.skipped ? [] : [result.llmJudge],
);
const passedResults = results.filter(isCasePassed);
return {
averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)),
@@ -563,16 +671,19 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary {
results.length === 0
? 0
: round(
results.filter((result) => result.ruleJudge.passed).length /
results.length,
passedResults.length / results.length,
),
passedCases: results.filter((result) => result.ruleJudge.passed).length,
passedCases: passedResults.length,
ruleAverageScore: averageOrZero(
results.map((result) => result.ruleJudge.score),
),
};
}
function isCasePassed(result: EvalCaseResult): boolean {
return result.ruleJudge.passed && (result.handoffJudge?.passed ?? true);
}
function createBaselineDelta(
current: EvalReportSummary,
baseline: EvalReportSummary,
@@ -717,16 +828,27 @@ async function evaluateCase(
input.roomContext && input.roomContext.length > 0
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
: undefined;
const deps = {
garnet: inMemoryGarnet,
llm: llmClient,
room: roomContextService,
};
if (skillName === "handoff") {
return evaluateHandoffCase(
evalCase,
input,
Object.values(SKILL_RUNNER_FACTORIES).map((factory) => factory(deps)),
llmClient,
enableLlmJudge,
);
}
const effectiveSkill = input.request.skill || skillName;
const factory = SKILL_RUNNER_FACTORIES[effectiveSkill];
if (!factory) {
throw new Error(`No eval runner factory for skill: ${effectiveSkill}`);
}
const skill = factory({
garnet: inMemoryGarnet,
llm: llmClient,
room: roomContextService,
});
const skill = factory(deps);
return runWithContext(
{
@@ -874,6 +996,105 @@ async function evaluateOnboardingCase(
});
}
async function evaluateHandoffCase(
evalCase: EvalCaseDefinition,
input: EvalCaseInput,
skills: readonly Skill[],
_llmClient: LlmClient,
_enableLlmJudge: boolean,
): Promise<EvalCaseResult> {
const startedAt = Date.now();
const derived: EvalDerivedSnapshot = {
allowShortResponse: true,
intent: "handoff",
phase: input.request.skill,
responseMode: "template",
shouldRespond: true,
suggestedMaxTokens: 192,
tier: 0,
};
try {
const result = await runEvalDispatch({
history: input.conversationHistory ?? [],
request: input.request,
skills,
});
const latencyMs = Date.now() - startedAt;
const handoffJudge = judgeHandoff(
evalCase.expected,
result.response,
result.finalSkill,
);
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: {
confidence: 1,
intent: "handoff",
source: "regex",
},
derived: {
...derived,
phase: result.finalSkill,
},
request: input.request,
response: result.response,
ruleJudge: judgeRuleCase(evalCase.expected, result.response.text, {
derived,
intent: "handoff",
responseMode: "template",
}),
llmJudge: {
skipped: true,
reason: "handoff-eval",
},
handoffJudge,
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
const errorMessage = error instanceof Error ? error.message : String(error);
const response: AgentResponse = {
confidence: 0,
metadata: {
error: errorMessage,
},
requestId: input.request.requestId,
source: "template",
text: "",
};
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: {
confidence: 0,
intent: "handoff",
source: "regex",
},
derived,
request: input.request,
response,
ruleJudge: judgeRuleCase(evalCase.expected, "", {
derived,
intent: "handoff",
responseMode: "template",
}),
llmJudge: {
skipped: true,
reason: "handoff-error",
},
handoffJudge: judgeHandoff(evalCase.expected, response, ""),
error: errorMessage,
};
}
}
async function evaluateGenericSkillCase(
evalCase: EvalCaseDefinition,
input: EvalCaseInput,
+33
View File
@@ -2,6 +2,7 @@ import type {
AgentRequest,
AgentResponse,
ConversationMessage,
HandoffReason,
} from "@agent/types";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
@@ -30,6 +31,16 @@ export interface EvalCaseInput {
}
export interface EvalCaseExpected {
expectedFinalSkill?: string | undefined;
expectedHandoff?: {
expectedReason?: HandoffReason | undefined;
expectedTarget?: string | undefined;
shouldHandoff?: boolean | undefined;
} | undefined;
expectedSourceChain?: {
chain: string[];
mode: "contains" | "exact" | "prefix";
} | undefined;
intent?: string | undefined;
mustIncludeAny?: string[] | undefined;
mustNotInclude?: string[] | undefined;
@@ -121,6 +132,27 @@ export interface ProcessJudgeResult {
totalCount: number;
}
export interface HandoffJudgeCheck {
detail?: string | undefined;
name:
| "chain"
| "final-skill"
| "handoff-reason"
| "handoff-target"
| "should-handoff";
passed: boolean;
skipped: boolean;
}
export interface HandoffJudgeResult {
actualChain: string[];
actualFinalSkill: string;
actualHandoffReason?: HandoffReason | undefined;
actualHandoffTarget?: string | undefined;
checks: HandoffJudgeCheck[];
passed: boolean;
}
export interface EvalDerivedSnapshot {
abstentionScore?: number | undefined;
allowShortResponse: boolean;
@@ -150,6 +182,7 @@ export interface EvalCaseResult {
response: AgentResponse;
ruleJudge: RuleJudgeResult;
llmJudge: LlmJudgeResult;
handoffJudge?: HandoffJudgeResult | undefined;
processJudge?: ProcessJudgeResult | undefined;
processTrace?: ProcessTraceSnapshot | undefined;
error?: string | undefined;
+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import {
formatReviewList,
formatReviewShow,
formatReviewStats,
} from "./cli.js";
describe("operations review CLI formatters", () => {
it("formats review list rows as tab-separated output", () => {
expect(
formatReviewList([
{
chatroomId: "room-1",
id: "1-0",
queuedAt: "2026-05-18T00:00:00.000Z",
reason: "low-confidence",
requestId: "request-1",
target: "human-review",
userId: "user-1",
},
]),
).toContain("1-0\t2026-05-18T00:00:00.000Z\tlow-confidence");
});
it("formats packet details for show output", () => {
expect(
formatReviewShow({
chatroomId: "room-1",
id: "1-0",
packet: {
constraints: [],
currentState: {},
evidence: [
{
id: "e1",
kind: "message",
summary: "unclear message",
},
],
expectedOutputSchema: "text",
handoffReason: "low-confidence",
sourceChain: ["onboarding", "supervisor"],
summary: "needs review",
userGoal: "의도 확인",
},
queuedAt: "2026-05-18T00:00:00.000Z",
reason: "low-confidence",
requestId: "request-1",
target: "human-review",
userId: "user-1",
}),
).toContain("sourceChain: onboarding > supervisor");
});
it("formats aggregate stats", () => {
expect(
formatReviewStats({
averagePendingAgeMs: 60_000,
byReason: {
"low-confidence": {
pending: 1,
resolved: 2,
total: 3,
},
},
pending: 1,
resolved: 2,
topSourceChains: [{ chain: "onboarding>supervisor", count: 3 }],
total: 3,
}),
).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2");
});
});
+299
View File
@@ -0,0 +1,299 @@
import { pathToFileURL } from "node:url";
import type { HandoffReason } from "@agent/types";
import { loadEnvironmentFromNodeEnv } from "@common/env-loader";
import { connectGarnetForCli } from "@common/garnet-cli";
import { initLogger } from "@obs/logger";
import {
getSupervisorReview,
getSupervisorReviewStats,
listSupervisorReview,
resolveSupervisorReview,
type SupervisorReviewAction,
type SupervisorReviewItem,
type SupervisorReviewStats,
} from "@skills/supervisor/review-queue";
import {
getFlagString,
hasFlag,
parseArgs,
parseDurationMs,
parsePositiveInteger,
} from "../eval/cli-args.js";
import { getReviewerId } from "../eval/review/cli.js";
function isHandoffReason(value: string | undefined): value is HandoffReason {
return (
value === "out-of-scope" ||
value === "missing-tool" ||
value === "low-confidence" ||
value === "validation-failed" ||
value === "same-agent-revisit" ||
value === "max-hop" ||
value === "high-risk-action" ||
value === "policy-risk"
);
}
function isSupervisorReviewAction(
value: string | undefined,
): value is SupervisorReviewAction {
return value === "manual-reply" || value === "reroute" || value === "dismiss";
}
function sinceDurationToIso(value: string | undefined): string | undefined {
if (!value?.trim()) {
return undefined;
}
return new Date(Date.now() - parseDurationMs(value)).toISOString();
}
function stringifyOptional(value: string | undefined): string {
return value?.trim() ? value : "-";
}
function formatAge(ms: number): string {
if (!Number.isFinite(ms) || ms <= 0) {
return "0s";
}
const seconds = Math.round(ms / 1000);
if (seconds < 60) {
return `${seconds}s`;
}
const minutes = Math.round(seconds / 60);
if (minutes < 60) {
return `${minutes}m`;
}
const hours = Math.round(minutes / 60);
if (hours < 48) {
return `${hours}h`;
}
return `${Math.round(hours / 24)}d`;
}
export function formatReviewList(items: SupervisorReviewItem[]): string {
const rows = [
[
"id",
"queuedAt",
"reason",
"chatroomId",
"requestId",
"target",
"resolved",
],
...items.map((item) => [
item.id,
item.queuedAt,
item.reason,
item.chatroomId,
item.requestId,
item.target,
item.resolution ? "yes" : "no",
]),
];
return rows.map((row) => row.join("\t")).join("\n");
}
export function formatReviewShow(item: SupervisorReviewItem): string {
const packet = item.packet;
const evidence =
packet?.evidence
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
.join("\n") || "-";
return [
`id: ${item.id}`,
`queuedAt: ${item.queuedAt}`,
`reason: ${item.reason}`,
`target: ${item.target}`,
`chatroomId: ${item.chatroomId}`,
`userId: ${item.userId}`,
`requestId: ${item.requestId}`,
`sourceChain: ${packet?.sourceChain.join(" > ") || "-"}`,
`userGoal: ${stringifyOptional(packet?.userGoal)}`,
`summary: ${stringifyOptional(packet?.summary)}`,
"[evidence]",
evidence,
item.resolution
? [
"[resolution]",
`action: ${item.resolution.action}`,
`resolvedAt: ${item.resolution.resolvedAt}`,
`resolvedBy: ${item.resolution.resolvedBy}`,
`note: ${stringifyOptional(item.resolution.note)}`,
].join("\n")
: "[resolution]\n-",
].join("\n");
}
export function formatReviewStats(stats: SupervisorReviewStats): string {
const reasonRows = Object.entries(stats.byReason)
.sort(([left], [right]) => left.localeCompare(right))
.map(
([reason, counts]) =>
`${reason}\ttotal=${counts.total}\tpending=${counts.pending}\tresolved=${counts.resolved}`,
);
const sourceRows = stats.topSourceChains.map(
(item) => `${item.chain}\tcount=${item.count}`,
);
return [
`total=${stats.total} pending=${stats.pending} resolved=${stats.resolved} averagePendingAge=${formatAge(stats.averagePendingAgeMs)}`,
"[byReason]",
...(reasonRows.length > 0 ? reasonRows : ["-"]),
"[topSourceChains]",
...(sourceRows.length > 0 ? sourceRows : ["-"]),
].join("\n");
}
async function handleReviewList(
flags: Map<string, string | true>,
): Promise<void> {
const reasonRaw = getFlagString(flags, "reason");
if (reasonRaw && !isHandoffReason(reasonRaw)) {
throw new Error(`Invalid --reason: ${reasonRaw}`);
}
const reason = isHandoffReason(reasonRaw) ? reasonRaw : undefined;
const connection = await connectGarnetForCli();
try {
const items = await listSupervisorReview(connection.garnetClient, {
includeResolved: hasFlag(flags, "include-resolved"),
limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? 50,
reason,
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since")),
untilQueuedAt: getFlagString(flags, "until"),
});
console.info(
hasFlag(flags, "json")
? JSON.stringify(items, null, 2)
: formatReviewList(items),
);
} finally {
await connection.close();
}
}
async function handleReviewShow(
id: string | undefined,
flags: Map<string, string | true>,
): Promise<void> {
if (!id?.trim()) {
throw new Error("review:show requires an entry id");
}
const connection = await connectGarnetForCli();
try {
const item = await getSupervisorReview(connection.garnetClient, id);
if (!item) {
process.exitCode = 2;
console.error(`[review] entry not found: ${id}`);
return;
}
console.info(
hasFlag(flags, "json")
? JSON.stringify(item, null, 2)
: formatReviewShow(item),
);
} finally {
await connection.close();
}
}
async function handleReviewResolve(
id: string | undefined,
flags: Map<string, string | true>,
): Promise<void> {
if (!id?.trim()) {
throw new Error("review:resolve requires an entry id");
}
const action = getFlagString(flags, "action");
if (!isSupervisorReviewAction(action)) {
throw new Error(
"--action is required and must be manual-reply, reroute, or dismiss",
);
}
const connection = await connectGarnetForCli();
try {
const item = await getSupervisorReview(connection.garnetClient, id);
if (!item) {
process.exitCode = 2;
console.error(`[review] entry not found: ${id}`);
return;
}
await resolveSupervisorReview(connection.garnetClient, id, {
action,
resolvedAt: new Date().toISOString(),
resolvedBy: getFlagString(flags, "by") ?? getReviewerId(),
...(getFlagString(flags, "note")
? { note: getFlagString(flags, "note") }
: {}),
});
console.info(`[review] resolved id=${id} action=${action}`);
} finally {
await connection.close();
}
}
async function handleReviewStats(
flags: Map<string, string | true>,
): Promise<void> {
const connection = await connectGarnetForCli();
try {
const stats = await getSupervisorReviewStats(connection.garnetClient, {
limit: parsePositiveInteger(getFlagString(flags, "limit")),
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since") ?? "7d"),
});
console.info(
hasFlag(flags, "json")
? JSON.stringify(stats, null, 2)
: formatReviewStats(stats),
);
} finally {
await connection.close();
}
}
async function main(): Promise<void> {
loadEnvironmentFromNodeEnv();
initLogger();
const { command, flags, positionals } = parseArgs(process.argv.slice(2));
if (command === "review:list") {
await handleReviewList(flags);
return;
}
if (command === "review:show") {
await handleReviewShow(positionals[0], flags);
return;
}
if (command === "review:resolve") {
await handleReviewResolve(positionals[0], flags);
return;
}
if (command === "review:stats") {
await handleReviewStats(flags);
return;
}
throw new Error(
"Unknown command. Use review:list, review:show, review:resolve, or review:stats.",
);
}
function isDirectCliEntry(): boolean {
const entrypoint = process.argv[1];
return entrypoint
? import.meta.url === pathToFileURL(entrypoint).href
: false;
}
if (isDirectCliEntry()) {
main().catch((error) => {
process.exitCode = process.exitCode || 1;
console.error(error instanceof Error ? error.message : String(error));
});
}
+170
View File
@@ -0,0 +1,170 @@
import { describe, expect, it } from "vitest";
import type { StreamEntry } from "@common/garnet";
import {
getSupervisorReview,
getSupervisorReviewStats,
listSupervisorReview,
parseSupervisorReviewEntry,
resolveSupervisorReview,
SUPERVISOR_RESOLVED_HASH_PREFIX,
SUPERVISOR_REVIEW_STREAM_KEY,
} from "./review-queue.js";
class FakeGarnet {
readonly hashes = new Map<string, Record<string, string>>();
readonly stream: StreamEntry[];
constructor(stream: StreamEntry[]) {
this.stream = stream;
}
readStreamRange(
key: string,
start: string,
end: string,
options?: { count?: number | undefined },
): Promise<StreamEntry[]> {
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
const entries = this.stream.filter((entry) => {
const afterStart = start === "-" || entry.id >= start;
const beforeEnd = end === "+" || entry.id <= end;
return afterStart && beforeEnd;
});
return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries);
}
getHash(key: string): Promise<Record<string, string>> {
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
}
setHash(key: string, fields: Record<string, string>): Promise<void> {
this.hashes.set(key, { ...fields });
return Promise.resolve();
}
expireKey(): Promise<boolean> {
return Promise.resolve(true);
}
}
const packet = {
constraints: [],
currentState: {},
evidence: [],
expectedOutputSchema: "text",
handoffReason: "out-of-scope",
sourceChain: ["onboarding", "moderation"],
summary: "user asked about ads",
userGoal: "광고 규칙 문의",
};
const entries: StreamEntry[] = [
{
id: "1-0",
message: {
chatroomId: "room-1",
packet: JSON.stringify(packet),
queuedAt: "2026-05-18T00:00:00.000Z",
reason: "out-of-scope",
requestId: "request-1",
target: "moderation",
type: "supervisor-handoff",
userId: "user-1",
},
},
{
id: "2-0",
message: {
chatroomId: "room-2",
queuedAt: "2026-05-18T01:00:00.000Z",
reason: "low-confidence",
requestId: "request-2",
target: "human-review",
type: "supervisor-handoff",
userId: "user-2",
},
},
];
describe("supervisor review queue", () => {
const firstEntry = entries[0]!;
it("parses a stream entry with the preserved handoff packet", () => {
expect(parseSupervisorReviewEntry(firstEntry)).toMatchObject({
chatroomId: "room-1",
id: "1-0",
packet,
reason: "out-of-scope",
target: "moderation",
});
});
it("lists pending entries and filters resolved items by default", async () => {
const garnet = new FakeGarnet(entries);
await resolveSupervisorReview(garnet, "1-0", {
action: "manual-reply",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await expect(listSupervisorReview(garnet)).resolves.toHaveLength(1);
await expect(
listSupervisorReview(garnet, { includeResolved: true, reason: "out-of-scope" }),
).resolves.toMatchObject([
{
id: "1-0",
resolution: {
action: "manual-reply",
resolvedBy: "operator@example.com",
},
},
]);
});
it("gets a single entry with resolution metadata", async () => {
const garnet = new FakeGarnet(entries);
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
action: "dismiss",
note: "duplicate",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await expect(getSupervisorReview(garnet, "1-0")).resolves.toMatchObject({
id: "1-0",
resolution: {
action: "dismiss",
note: "duplicate",
},
});
});
it("computes reason totals and top source chains", async () => {
const garnet = new FakeGarnet(entries);
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
action: "reroute",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await expect(getSupervisorReviewStats(garnet)).resolves.toMatchObject({
byReason: {
"low-confidence": {
pending: 1,
resolved: 0,
total: 1,
},
"out-of-scope": {
pending: 0,
resolved: 1,
total: 1,
},
},
pending: 1,
resolved: 1,
topSourceChains: [{ chain: "onboarding>moderation", count: 1 }],
total: 2,
});
});
});
+336 -6
View File
@@ -1,14 +1,344 @@
import type { GarnetClient } from "@common/garnet";
import type { HandoffPacket } from "@agent/handoff-packet";
import type { HandoffReason } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet";
import {
getSupervisorReviewQueueMaxLen,
getSupervisorReviewResolutionTtlSec,
} from "../../config.js";
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
"lolympus-agent:review-resolved:";
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
export interface SupervisorReviewEntry {
id: string;
queuedAt: string;
chatroomId: string;
userId: string;
requestId: string;
reason: HandoffReason;
target: string;
packet?: HandoffPacket | undefined;
extra?: Record<string, string> | undefined;
}
export interface SupervisorReviewResolution {
resolvedAt: string;
resolvedBy: string;
action: SupervisorReviewAction;
note?: string | undefined;
}
export interface SupervisorReviewItem extends SupervisorReviewEntry {
resolution?: SupervisorReviewResolution | undefined;
}
export interface ListSupervisorReviewOptions {
includeResolved?: boolean | undefined;
limit?: number | undefined;
reason?: HandoffReason | undefined;
since?: string | undefined;
sinceQueuedAt?: string | undefined;
until?: string | undefined;
untilQueuedAt?: string | undefined;
}
export interface SupervisorReviewReasonStats {
pending: number;
resolved: number;
total: number;
}
export interface SupervisorReviewStats {
averagePendingAgeMs: number;
byReason: Record<string, SupervisorReviewReasonStats>;
pending: number;
resolved: number;
topSourceChains: Array<{ chain: string; count: number }>;
total: number;
}
type SupervisorReviewListGarnetClient = Pick<
GarnetClient,
"getHash" | "readStreamRange"
>;
type SupervisorReviewResolveGarnetClient = Pick<
GarnetClient,
"expireKey" | "setHash"
>;
const BASE_FIELDS = new Set([
"chatroomId",
"packet",
"queuedAt",
"reason",
"requestId",
"target",
"type",
"userId",
]);
function isHandoffReason(value: string | undefined): value is HandoffReason {
return (
value === "out-of-scope" ||
value === "missing-tool" ||
value === "low-confidence" ||
value === "validation-failed" ||
value === "same-agent-revisit" ||
value === "max-hop" ||
value === "high-risk-action" ||
value === "policy-risk"
);
}
function isSupervisorReviewAction(
value: string | undefined,
): value is SupervisorReviewAction {
return value === "manual-reply" || value === "reroute" || value === "dismiss";
}
function parseOptionalPacket(value: string | undefined): HandoffPacket | undefined {
if (!value?.trim() || value.trim() === "null") {
return undefined;
}
try {
const parsed = JSON.parse(value) as unknown;
return typeof parsed === "object" && parsed !== null
? (parsed as HandoffPacket)
: undefined;
} catch {
return undefined;
}
}
function parseResolution(
fields: Record<string, string>,
): SupervisorReviewResolution | undefined {
if (
!fields["resolvedAt"] ||
!fields["resolvedBy"] ||
!isSupervisorReviewAction(fields["action"])
) {
return undefined;
}
return {
action: fields["action"],
resolvedAt: fields["resolvedAt"],
resolvedBy: fields["resolvedBy"],
...(fields["note"] ? { note: fields["note"] } : {}),
};
}
function resolutionKey(id: string): string {
return `${SUPERVISOR_RESOLVED_HASH_PREFIX}${id}`;
}
function isAfterOrEqual(value: string, lowerBound: string | undefined): boolean {
return !lowerBound || value >= lowerBound;
}
function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean {
return !upperBound || value <= upperBound;
}
export async function appendSupervisorReview(
garnetClient: Pick<GarnetClient, "appendStream">,
fields: Record<string, string>,
): Promise<void> {
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
await garnetClient.appendStream(
SUPERVISOR_REVIEW_STREAM_KEY,
{
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
},
{ maxLen: getSupervisorReviewQueueMaxLen() },
);
}
export function parseSupervisorReviewEntry(
streamEntry: StreamEntry,
): SupervisorReviewEntry | null {
const { message } = streamEntry;
const reason = message["reason"];
if (
!message["queuedAt"] ||
!message["chatroomId"] ||
!message["userId"] ||
!message["requestId"] ||
!message["target"] ||
!isHandoffReason(reason)
) {
return null;
}
const extra = Object.fromEntries(
Object.entries(message).filter(([key]) => !BASE_FIELDS.has(key)),
);
const packet = parseOptionalPacket(message["packet"]);
return {
chatroomId: message["chatroomId"],
id: streamEntry.id,
queuedAt: message["queuedAt"],
reason,
requestId: message["requestId"],
target: message["target"],
userId: message["userId"],
...(packet ? { packet } : {}),
...(Object.keys(extra).length > 0 ? { extra } : {}),
};
}
export async function getSupervisorReview(
garnetClient: SupervisorReviewListGarnetClient,
id: string,
): Promise<SupervisorReviewItem | null> {
const entries = await garnetClient.readStreamRange(
SUPERVISOR_REVIEW_STREAM_KEY,
id,
id,
{ count: 1 },
);
const parsed = entries.flatMap((entry) => {
const review = parseSupervisorReviewEntry(entry);
return review ? [review] : [];
})[0];
if (!parsed) {
return null;
}
const resolution = parseResolution(
await garnetClient.getHash(resolutionKey(parsed.id)),
);
return {
...parsed,
...(resolution ? { resolution } : {}),
};
}
export async function listSupervisorReview(
garnetClient: SupervisorReviewListGarnetClient,
options?: ListSupervisorReviewOptions,
): Promise<SupervisorReviewItem[]> {
const entries = await garnetClient.readStreamRange(
SUPERVISOR_REVIEW_STREAM_KEY,
options?.since ?? "-",
options?.until ?? "+",
);
const items: SupervisorReviewItem[] = [];
for (const entry of entries) {
const review = parseSupervisorReviewEntry(entry);
if (!review) {
continue;
}
if (options?.reason && review.reason !== options.reason) {
continue;
}
if (!isAfterOrEqual(review.queuedAt, options?.sinceQueuedAt)) {
continue;
}
if (!isBeforeOrEqual(review.queuedAt, options?.untilQueuedAt)) {
continue;
}
const resolution = parseResolution(
await garnetClient.getHash(resolutionKey(review.id)),
);
if (resolution && !options?.includeResolved) {
continue;
}
items.push({
...review,
...(resolution ? { resolution } : {}),
});
if (options?.limit && items.length >= options.limit) {
break;
}
}
return items;
}
export async function resolveSupervisorReview(
garnetClient: SupervisorReviewResolveGarnetClient,
id: string,
resolution: SupervisorReviewResolution,
): Promise<void> {
const fields: Record<string, string> = {
action: resolution.action,
resolvedAt: resolution.resolvedAt,
resolvedBy: resolution.resolvedBy,
};
if (resolution.note?.trim()) {
fields.note = resolution.note.trim();
}
const key = resolutionKey(id);
await garnetClient.setHash(key, fields);
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
}
export async function getSupervisorReviewStats(
garnetClient: SupervisorReviewListGarnetClient,
options?: Omit<ListSupervisorReviewOptions, "includeResolved">,
): Promise<SupervisorReviewStats> {
const nowMs = Date.now();
const items = await listSupervisorReview(garnetClient, {
...options,
includeResolved: true,
});
const byReason: Record<string, SupervisorReviewReasonStats> = {};
const sourceChains = new Map<string, number>();
let pendingAgeMs = 0;
let pending = 0;
let resolved = 0;
for (const item of items) {
const reasonStats = byReason[item.reason] ?? {
pending: 0,
resolved: 0,
total: 0,
};
reasonStats.total += 1;
if (item.resolution) {
resolved += 1;
reasonStats.resolved += 1;
} else {
pending += 1;
reasonStats.pending += 1;
const queuedAtMs = Date.parse(item.queuedAt);
if (Number.isFinite(queuedAtMs)) {
pendingAgeMs += Math.max(0, nowMs - queuedAtMs);
}
}
byReason[item.reason] = reasonStats;
const chain = item.packet?.sourceChain?.join(">");
if (chain) {
sourceChains.set(chain, (sourceChains.get(chain) ?? 0) + 1);
}
}
return {
averagePendingAgeMs: pending === 0 ? 0 : Math.round(pendingAgeMs / pending),
byReason,
pending,
resolved,
topSourceChains: [...sourceChains.entries()]
.map(([chain, count]) => ({ chain, count }))
.sort((left, right) => right.count - left.count)
.slice(0, 5),
total: items.length,
};
}
+9
View File
@@ -82,6 +82,15 @@ describe("SupervisorSkill", () => {
type: "supervisor-handoff",
},
});
expect(garnet.streams[0]?.fields["packet"]).toEqual(expect.any(String));
const queuedPacket: unknown = JSON.parse(
String(garnet.streams[0]?.fields["packet"]),
);
expect(queuedPacket).toMatchObject({
handoffReason: "max-hop",
sourceChain: ["onboarding", "supervisor"],
userGoal: "가입 상태 확인",
});
expect(nats.published[0]).toMatchObject({
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
+1
View File
@@ -56,6 +56,7 @@ export class SupervisorSkill implements Skill {
await appendSupervisorReview(this.garnetClient, {
chatroomId: request.chatroomId,
packet: JSON.stringify(packet ?? null),
reason,
requestId: request.requestId,
target: target ?? "human-review",