From f0b00e5fd22c7758522c8fea5e7f935341416b76 Mon Sep 17 00:00:00 2001 From: dahyeon Date: Mon, 18 May 2026 18:02:13 -0700 Subject: [PATCH] Close supervisor review loop and add handoff eval --- package.json | 5 + src/agent/agent.test.ts | 16 + src/agent/agent.ts | 28 +- src/common/garnet-cli.ts | 13 + src/config.ts | 11 + src/eval/cli-args.ts | 113 ++++++ src/eval/cli.ts | 117 +----- src/eval/golden/handoff/cases.jsonl | 6 + src/eval/handoff-dispatch.ts | 412 +++++++++++++++++++++ src/eval/judges/handoff-judge.test.ts | 102 +++++ src/eval/judges/handoff-judge.ts | 152 ++++++++ src/eval/review/cli.ts | 2 +- src/eval/runner.ts | 241 +++++++++++- src/eval/types.ts | 33 ++ src/operations/cli.test.ts | 74 ++++ src/operations/cli.ts | 299 +++++++++++++++ src/skills/supervisor/review-queue.test.ts | 170 +++++++++ src/skills/supervisor/review-queue.ts | 342 ++++++++++++++++- src/skills/supervisor/skill.test.ts | 9 + src/skills/supervisor/skill.ts | 1 + 20 files changed, 2017 insertions(+), 129 deletions(-) create mode 100644 src/common/garnet-cli.ts create mode 100644 src/eval/cli-args.ts create mode 100644 src/eval/golden/handoff/cases.jsonl create mode 100644 src/eval/handoff-dispatch.ts create mode 100644 src/eval/judges/handoff-judge.test.ts create mode 100644 src/eval/judges/handoff-judge.ts create mode 100644 src/operations/cli.test.ts create mode 100644 src/operations/cli.ts create mode 100644 src/skills/supervisor/review-queue.test.ts diff --git a/package.json b/package.json index cee07c5..2c0f1d3 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "format": "prettier --write .", "eval": "node --import tsx src/eval/cli.ts run --set=onboarding/cases", "eval:moderation": "node --import tsx src/eval/cli.ts run --set=moderation/cases", + "eval:handoff": "node --import tsx src/eval/cli.ts run --set=handoff/cases", "eval:capture": "node --import tsx src/eval/cli.ts capture", "eval:baseline": "node --import tsx src/eval/cli.ts baseline", "eval:report": "node --import tsx src/eval/cli.ts report", @@ -28,6 +29,10 @@ "eval:tournament": "node --import tsx src/eval/cli.ts tournament", "eval:tournament:moderation": "node --import tsx src/eval/cli.ts tournament --skill=moderation", "eval:tournament-results": "node --import tsx src/eval/cli.ts tournament-results", + "review:list": "node --import tsx src/operations/cli.ts review:list", + "review:show": "node --import tsx src/operations/cli.ts review:show", + "review:resolve": "node --import tsx src/operations/cli.ts review:resolve", + "review:stats": "node --import tsx src/operations/cli.ts review:stats", "start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"", "start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"", "start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"", diff --git a/src/agent/agent.test.ts b/src/agent/agent.test.ts index 41789d7..a344593 100644 --- a/src/agent/agent.test.ts +++ b/src/agent/agent.test.ts @@ -151,6 +151,10 @@ describe("Agent handoff dispatch", () => { expect(nats.responses).toHaveLength(1); expect(nats.responses[0]?.text).toBe("final answer"); + expect(nats.responses[0]?.metadata).toMatchObject({ + handoffChain: ["source", "target"], + handoffDepth: 1, + }); expect(store.messages.map((message) => message.content)).toEqual([ "final answer", ]); @@ -189,6 +193,10 @@ describe("Agent handoff dispatch", () => { expect(nats.responses[0]?.text).toBe("handled elsewhere"); expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true); + expect(nats.responses[0]?.metadata).toMatchObject({ + handoffChain: ["source", "target"], + handoffDepth: 1, + }); }); it("routes runtime verifier escalations to supervisor", async () => { @@ -228,6 +236,10 @@ describe("Agent handoff dispatch", () => { expect(nats.responses[0]?.text).toBe("sent to review"); expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true); + expect(nats.responses[0]?.metadata).toMatchObject({ + handoffChain: ["moderation", "supervisor"], + handoffDepth: 1, + }); } finally { if (previous === undefined) { delete process.env["AGENT_RUNTIME_VERIFIER_ENABLED"]; @@ -297,5 +309,9 @@ describe("Agent handoff dispatch", () => { expect(nats.responses[0]?.text).toBe("supervisor review"); expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true); + expect(nats.responses[0]?.metadata).toMatchObject({ + handoffChain: ["source", "supervisor"], + handoffDepth: 1, + }); }); }); diff --git a/src/agent/agent.ts b/src/agent/agent.ts index 59ee6d9..744b1ae 100644 --- a/src/agent/agent.ts +++ b/src/agent/agent.ts @@ -282,10 +282,11 @@ export class Agent { response: verified, skill, }); + const annotated = this.withHandoffMetadata(escalated, chain, depth); - const handoff = readHandoffRequest(escalated); + const handoff = readHandoffRequest(annotated); if (!handoff) { - return escalated; + return annotated; } const next = this.prepareHandoff({ @@ -296,11 +297,15 @@ export class Agent { handoff, history, request, - response: escalated, + response: annotated, }); if (!next) { - return this.makeSupervisorFallback(request, handoff.reason); + return this.withHandoffMetadata( + this.makeSupervisorFallback(request, handoff.reason), + [...chain, "supervisor"], + depth + 1, + ); } return this.dispatch(next); @@ -308,6 +313,21 @@ export class Agent { ); } + private withHandoffMetadata( + response: AgentResponse, + chain: string[], + depth: number, + ): AgentResponse { + return { + ...response, + metadata: { + ...(response.metadata ?? {}), + handoffChain: chain, + handoffDepth: depth, + }, + }; + } + private async runSkillOnce(input: { chain: string[]; depth: number; diff --git a/src/common/garnet-cli.ts b/src/common/garnet-cli.ts new file mode 100644 index 0000000..b141f57 --- /dev/null +++ b/src/common/garnet-cli.ts @@ -0,0 +1,13 @@ +import { GarnetClient } from "./garnet.js"; + +export async function connectGarnetForCli(): Promise<{ + close: () => Promise; + garnetClient: GarnetClient; +}> { + const garnetClient = new GarnetClient(); + await garnetClient.connect(); + return { + close: () => garnetClient.close(), + garnetClient, + }; +} diff --git a/src/config.ts b/src/config.ts index 02f4939..7a9193b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -218,6 +218,17 @@ export function getEvalReviewDedupeTtlSec(): number { return getPositiveNumber("EVAL_REVIEW_DEDUPE_TTL_SEC", 604_800); } +export function getSupervisorReviewQueueMaxLen(): number { + return getPositiveNumber("SUPERVISOR_REVIEW_QUEUE_MAXLEN", 10_000); +} + +export function getSupervisorReviewResolutionTtlSec(): number { + return getPositiveNumber( + "SUPERVISOR_REVIEW_RESOLUTION_TTL_SEC", + 90 * 24 * 60 * 60, + ); +} + export function getEvalCalibrationMinCorr(): number { return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5); } diff --git a/src/eval/cli-args.ts b/src/eval/cli-args.ts new file mode 100644 index 0000000..d3b07a1 --- /dev/null +++ b/src/eval/cli-args.ts @@ -0,0 +1,113 @@ +export interface ParsedArgs { + command: string; + flags: Map; + positionals: string[]; +} + +export function parsePositiveInteger( + value: string | undefined, +): number | undefined { + if (!value?.trim()) { + return undefined; + } + + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; +} + +export function parsePositiveNumber( + value: string | undefined, +): number | undefined { + if (!value?.trim()) { + return undefined; + } + + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + +export function parseDurationMs(value: string | undefined): number { + const raw = value?.trim().toLowerCase() || "7d"; + const match = raw.match(/^(\d+)(h|d)$/u); + if (!match) { + throw new Error("--since must use a duration like 24h, 7d, or 30d"); + } + + const amount = Number(match[1]); + const unit = match[2]; + if (!Number.isInteger(amount) || amount <= 0) { + throw new Error("--since must be a positive duration"); + } + + return unit === "h" + ? amount * 60 * 60 * 1000 + : amount * 24 * 60 * 60 * 1000; +} + +export function parseCsv(value: string | undefined): string[] { + if (!value?.trim()) { + return []; + } + + return value + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); +} + +export function parseArgs(argv: string[]): ParsedArgs { + const [command = "run", ...rest] = argv; + const flags = new Map(); + const positionals: string[] = []; + + for (let index = 0; index < rest.length; index++) { + const token = rest[index]; + if (!token) { + continue; + } + + if (!token.startsWith("--")) { + positionals.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const equalsIndex = withoutPrefix.indexOf("="); + if (equalsIndex >= 0) { + const key = withoutPrefix.slice(0, equalsIndex); + const value = withoutPrefix.slice(equalsIndex + 1); + flags.set(key, value || true); + continue; + } + + const next = rest[index + 1]; + if (next && !next.startsWith("--")) { + flags.set(withoutPrefix, next); + index += 1; + continue; + } + + flags.set(withoutPrefix, true); + } + + return { + command, + flags, + positionals, + }; +} + +export function getFlagString( + flags: Map, + key: string, +): string | undefined { + const value = flags.get(key); + return typeof value === "string" ? value : undefined; +} + +export function hasFlag( + flags: Map, + key: string, +): boolean { + return flags.has(key); +} diff --git a/src/eval/cli.ts b/src/eval/cli.ts index 5b5439a..03e9c41 100644 --- a/src/eval/cli.ts +++ b/src/eval/cli.ts @@ -1,51 +1,15 @@ import type { MongoConnectionLike } from "./archive.js"; import type { SliceKey, SliceReportRow } from "./aggregator.js"; import type { GarnetClient } from "@common/garnet"; - -function parsePositiveInteger(value: string | undefined): number | undefined { - if (!value?.trim()) { - return undefined; - } - - const parsed = Number(value); - return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; -} - -function parsePositiveNumber(value: string | undefined): number | undefined { - if (!value?.trim()) { - return undefined; - } - - const parsed = Number(value); - return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; -} - -function parseDurationMs(value: string | undefined): number { - const raw = value?.trim().toLowerCase() || "7d"; - const match = raw.match(/^(\d+)(h|d)$/u); - if (!match) { - throw new Error("--since must use a duration like 24h, 7d, or 30d"); - } - - const amount = Number(match[1]); - const unit = match[2]; - if (!Number.isInteger(amount) || amount <= 0) { - throw new Error("--since must be a positive duration"); - } - - return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000; -} - -function parseCsv(value: string | undefined): string[] { - if (!value?.trim()) { - return []; - } - - return value - .split(",") - .map((item) => item.trim()) - .filter((item) => item.length > 0); -} +import { + getFlagString, + hasFlag, + parseArgs, + parseCsv, + parseDurationMs, + parsePositiveInteger, + parsePositiveNumber, +} from "./cli-args.js"; function parseSlice(value: string | undefined): SliceKey { if (!value?.trim()) { @@ -85,69 +49,6 @@ function parseSlice(value: string | undefined): SliceKey { }; } -interface ParsedArgs { - command: string; - flags: Map; - positionals: string[]; -} - -function parseArgs(argv: string[]): ParsedArgs { - const [command = "run", ...rest] = argv; - const flags = new Map(); - const positionals: string[] = []; - - for (let index = 0; index < rest.length; index++) { - const token = rest[index]; - if (!token) { - continue; - } - - if (!token.startsWith("--")) { - positionals.push(token); - continue; - } - - const withoutPrefix = token.slice(2); - const equalsIndex = withoutPrefix.indexOf("="); - if (equalsIndex >= 0) { - const key = withoutPrefix.slice(0, equalsIndex); - const value = withoutPrefix.slice(equalsIndex + 1); - flags.set(key, value || true); - continue; - } - - const next = rest[index + 1]; - if (next && !next.startsWith("--")) { - flags.set(withoutPrefix, next); - index += 1; - continue; - } - - flags.set(withoutPrefix, true); - } - - return { - command, - flags, - positionals, - }; -} - -function getFlagString( - flags: Map, - key: string, -): string | undefined { - const value = flags.get(key); - return typeof value === "string" ? value : undefined; -} - -function hasFlag( - flags: Map, - key: string, -): boolean { - return flags.has(key); -} - function printRunSummary(report: { generatedAt: string; summary: { diff --git a/src/eval/golden/handoff/cases.jsonl b/src/eval/golden/handoff/cases.jsonl new file mode 100644 index 0000000..e867fc9 --- /dev/null +++ b/src/eval/golden/handoff/cases.jsonl @@ -0,0 +1,6 @@ +{"id":"case_handoff_onboarding_to_moderation_ad","name":"onboarding routes community rule question to moderation","input":{"request":{"requestId":"handoff-1","skill":"onboarding","chatroomId":"room-handoff","userId":"user-1","nickname":"tester","trigger":"user-message","userMessage":"광고 올려도 돼요?","context":{"evalHandoffPlan":[{"from":"onboarding","to":"moderation","reason":"out-of-scope","taskSpec":"광고/홍보 규칙 문의 처리"}]}},"conversationHistory":[{"role":"user","content":"광고 올려도 돼요?","timestamp":"2026-05-18T00:00:00.000Z"}]},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"out-of-scope","expectedTarget":"moderation"},"expectedFinalSkill":"moderation","expectedSourceChain":{"mode":"contains","chain":["moderation"]}}} +{"id":"case_handoff_moderation_to_onboarding_welcome","name":"moderation routes onboarding question to onboarding","input":{"request":{"requestId":"handoff-2","skill":"moderation","chatroomId":"room-handoff","userId":"user-2","nickname":"tester","trigger":"user-message","userMessage":"신규 가입 안내랑 닉네임 규칙 알려줘","context":{"evalHandoffPlan":[{"from":"moderation","to":"onboarding","reason":"out-of-scope","taskSpec":"신규 가입 안내 처리"}]}},"conversationHistory":[{"role":"user","content":"신규 가입 안내랑 닉네임 규칙 알려줘","timestamp":"2026-05-18T00:00:00.000Z"}]},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"out-of-scope","expectedTarget":"onboarding"},"expectedFinalSkill":"onboarding","expectedSourceChain":{"mode":"exact","chain":["moderation","onboarding"]}}} +{"id":"case_handoff_low_confidence_to_supervisor","name":"unclear handoff escalates to supervisor","input":{"request":{"requestId":"handoff-3","skill":"onboarding","chatroomId":"room-handoff","userId":"user-3","nickname":"tester","trigger":"user-message","userMessage":"음... 이거랑 저거 그거 어떻게 해?","context":{"evalHandoffPlan":[{"from":"onboarding","to":"supervisor","reason":"low-confidence","taskSpec":"의도 불명확 케이스 운영자 검토"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"low-confidence","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["onboarding","supervisor"]}}} +{"id":"case_handoff_validation_retry_then_supervisor","name":"validation failure escalates to supervisor","input":{"request":{"requestId":"handoff-4","skill":"moderation","chatroomId":"room-handoff","userId":"user-4","nickname":"tester","trigger":"user-message","userMessage":"규칙 답변에 링크를 넣어도 돼?","context":{"evalHandoffPlan":[{"from":"moderation","to":"supervisor","reason":"validation-failed","taskSpec":"runtime verifier failed after retry"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"validation-failed","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["moderation","supervisor"]}}} +{"id":"case_handoff_aba_loop_blocked","name":"aba loop is rerouted to supervisor","input":{"request":{"requestId":"handoff-5","skill":"onboarding","chatroomId":"room-handoff","userId":"user-5","nickname":"tester","trigger":"user-message","userMessage":"가입인지 규칙인지 헷갈리는 질문","context":{"evalBudgetMaxSameAgentVisits":1,"evalHandoffPlan":[{"from":"onboarding","to":"moderation","reason":"out-of-scope","taskSpec":"규칙 확인"},{"from":"moderation","to":"onboarding","reason":"out-of-scope","taskSpec":"가입 상태 확인"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"same-agent-revisit","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["onboarding","moderation","supervisor"]}}} +{"id":"case_handoff_high_risk_to_supervisor","name":"high risk action escalates to supervisor","input":{"request":{"requestId":"handoff-6","skill":"moderation","chatroomId":"room-handoff","userId":"user-6","nickname":"tester","trigger":"user-message","userMessage":"이 유저 바로 밴해줘","context":{"evalHandoffPlan":[{"from":"moderation","to":"supervisor","reason":"high-risk-action","risk":"high","taskSpec":"제재성 고위험 요청 운영자 검토"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"high-risk-action","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["moderation","supervisor"]}}} diff --git a/src/eval/handoff-dispatch.ts b/src/eval/handoff-dispatch.ts new file mode 100644 index 0000000..8b6ef06 --- /dev/null +++ b/src/eval/handoff-dispatch.ts @@ -0,0 +1,412 @@ +import { BudgetTracker, type BudgetVerdict } from "@agent/budget"; +import { buildHandoffPacket } from "@agent/handoff-packet"; +import { selectSkillForRequest } from "@agent/agent"; +import type { + AgentRequest, + AgentResponse, + ConversationMessage, + HandoffReason, + HandoffRequest, +} from "@agent/types"; +import { parseControlTokens } from "@skills/control-token-parser"; +import { decide as decideEscalation } from "@skills/escalation/trigger"; +import type { Skill } from "@skills/skill"; + +export interface EvalDispatchResult { + chain: string[]; + finalSkill: string; + response: AgentResponse; +} + +interface EvalHandoffPlanStep { + confidence?: number | undefined; + expectedOutput?: string | undefined; + from: string; + reason: HandoffReason; + risk?: HandoffRequest["risk"] | undefined; + taskSpec?: string | undefined; + to: string; +} + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null + ? (value as Record) + : {}; +} + +function readPlan(request: AgentRequest): EvalHandoffPlanStep[] { + const rawPlan = asRecord(request.context)["evalHandoffPlan"]; + if (!Array.isArray(rawPlan)) { + return []; + } + + return rawPlan.flatMap((item): EvalHandoffPlanStep[] => { + const record = asRecord(item); + const from = record["from"]; + const to = record["to"]; + const reason = record["reason"]; + if ( + typeof from !== "string" || + typeof to !== "string" || + !isHandoffReason(reason) + ) { + return []; + } + + return [ + { + confidence: + typeof record["confidence"] === "number" + ? record["confidence"] + : undefined, + expectedOutput: + typeof record["expectedOutput"] === "string" + ? record["expectedOutput"] + : undefined, + from, + reason, + risk: + record["risk"] === "low" || + record["risk"] === "medium" || + record["risk"] === "high" + ? record["risk"] + : undefined, + taskSpec: + typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined, + to, + }, + ]; + }); +} + +function isHandoffReason(value: unknown): value is HandoffReason { + return ( + value === "out-of-scope" || + value === "missing-tool" || + value === "low-confidence" || + value === "validation-failed" || + value === "same-agent-revisit" || + value === "max-hop" || + value === "high-risk-action" || + value === "policy-risk" + ); +} + +function readPositiveNumber( + context: Record | undefined, + key: string, + fallback: number, +): number { + const value = context?.[key]; + return typeof value === "number" && Number.isFinite(value) && value > 0 + ? value + : fallback; +} + +function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason { + if (verdict === "same-agent-loop") { + return "same-agent-revisit"; + } + if (verdict === "max-hops") { + return "max-hop"; + } + return "policy-risk"; +} + +function readHandoffRequest(response: AgentResponse): HandoffRequest | null { + const value = response.metadata?.["handoffRequest"]; + if (typeof value !== "object" || value === null) { + return null; + } + const request = value as Partial; + if ( + typeof request.targetAgent !== "string" || + typeof request.reason !== "string" || + typeof request.taskSpec !== "string" + ) { + return null; + } + + return { + confidence: + typeof request.confidence === "number" ? request.confidence : response.confidence, + evidenceRefs: Array.isArray(request.evidenceRefs) + ? request.evidenceRefs.filter((item): item is string => typeof item === "string") + : [], + expectedOutput: + typeof request.expectedOutput === "string" + ? request.expectedOutput + : "text", + reason: request.reason, + risk: + request.risk === "low" || request.risk === "medium" || request.risk === "high" + ? request.risk + : "medium", + targetAgent: request.targetAgent, + taskSpec: request.taskSpec, + ...(request.userVisibleMessage + ? { userVisibleMessage: request.userVisibleMessage } + : {}), + }; +} + +function normalizeControlTokens( + response: AgentResponse, + currentSkill: string, +): AgentResponse { + const parsed = parseControlTokens(response.text); + const handoff = parsed.tokens.find((token) => token.kind === "handoff"); + if (handoff?.kind !== "handoff") { + return { + ...response, + text: parsed.stripped, + }; + } + + return { + ...response, + metadata: { + ...(response.metadata ?? {}), + handoffRequest: { + confidence: response.confidence, + evidenceRefs: [], + expectedOutput: "text", + reason: handoff.reason, + risk: "medium", + targetAgent: handoff.targetAgent, + taskSpec: `Continue handling from ${currentSkill}`, + } satisfies HandoffRequest, + }, + text: parsed.stripped, + }; +} + +function withHandoffMetadata( + response: AgentResponse, + chain: string[], + depth: number, + lastHandoff: HandoffRequest | undefined, +): AgentResponse { + return { + ...response, + metadata: { + ...(response.metadata ?? {}), + handoffChain: chain, + handoffDepth: depth, + ...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}), + }, + }; +} + +function plannedHandoffResponse( + request: AgentRequest, + step: EvalHandoffPlanStep, +): AgentResponse { + return { + confidence: step.confidence ?? 0.72, + metadata: { + handoffRequest: { + confidence: step.confidence ?? 0.72, + evidenceRefs: [], + expectedOutput: step.expectedOutput ?? "text", + reason: step.reason, + risk: step.risk ?? "medium", + targetAgent: step.to, + taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`, + } satisfies HandoffRequest, + }, + requestId: request.requestId, + source: "template", + text: `handoff ${step.from} to ${step.to}`, + }; +} + +async function runActualSkill( + request: AgentRequest, + history: ConversationMessage[], + skill: Skill, + budget: BudgetTracker, +): Promise { + const response = normalizeControlTokens( + await skill.handle(request, history), + skill.name, + ); + if (!response.text.trim() || readHandoffRequest(response)) { + return response; + } + + const decision = decideEscalation({ + budget: budget.snapshot(), + capabilityCard: skill.capabilityCard?.(), + classified: { confidence: response.confidence }, + targetAgent: "supervisor", + taskSpec: `Review ${skill.name} response for ${request.trigger}.`, + }); + if (!decision.handoff) { + return response; + } + + return { + ...response, + metadata: { + ...(response.metadata ?? {}), + handoffRequest: decision.handoff, + }, + }; +} + +export async function runEvalDispatch(input: { + history?: ConversationMessage[] | undefined; + request: AgentRequest; + skills: readonly Skill[]; +}): Promise { + const context = input.request.context; + const plan = readPlan(input.request); + const useActualSkills = asRecord(context)["evalUseActualSkills"] === true; + const budget = new BudgetTracker({ + enabled: true, + maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5), + maxHandoffs: readPositiveNumber(context, "evalBudgetMaxHandoffs", 4), + maxSameAgentVisits: readPositiveNumber( + context, + "evalBudgetMaxSameAgentVisits", + 2, + ), + maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20), + }); + + let request = input.request; + let skill = selectSkillForRequest(input.skills, request); + if (!skill) { + throw new Error(`No skill can handle eval request: ${request.skill}`); + } + + let chain = [skill.name]; + let lastHandoff: HandoffRequest | undefined; + let planIndex = 0; + budget.recordVisit(skill.name); + + for (let depth = 0; depth <= 8; depth++) { + const planned = plan[planIndex]?.from === skill.name ? plan[planIndex] : undefined; + const response = planned + ? plannedHandoffResponse(request, planned) + : useActualSkills + ? await runActualSkill(request, input.history ?? [], skill, budget) + : { + confidence: 1, + requestId: request.requestId, + source: "template" as const, + text: `handoff eval final: ${skill.name}`, + }; + if (planned) { + planIndex += 1; + } + + const handoff = readHandoffRequest(response); + if (!handoff) { + const finalResponse = withHandoffMetadata( + response, + chain, + chain.length - 1, + lastHandoff, + ); + return { + chain, + finalSkill: skill.name, + response: finalResponse, + }; + } + + const verdict = budget.assertBeforeHandoff(handoff.targetAgent); + const resolved: HandoffRequest = + verdict === "ok" + ? handoff + : { + ...handoff, + reason: budgetVerdictToReason(verdict), + targetAgent: "supervisor", + }; + const nextSkill = selectSkillForRequest(input.skills, { + ...request, + skill: resolved.targetAgent, + trigger: "handoff", + }); + if (!nextSkill) { + const fallbackChain = [...chain, "supervisor"]; + const fallbackHandoff = { + ...resolved, + targetAgent: "supervisor", + }; + return { + chain: fallbackChain, + finalSkill: "supervisor", + response: withHandoffMetadata( + { + confidence: 0.4, + requestId: request.requestId, + source: "template", + text: "확인이 필요한 내용이라 운영자에게 전달했어요.", + }, + fallbackChain, + fallbackChain.length - 1, + fallbackHandoff, + ), + }; + } + + budget.recordHandoff(nextSkill.name); + const nextChain = [...chain, nextSkill.name]; + lastHandoff = { + ...resolved, + targetAgent: nextSkill.name, + }; + const packet = buildHandoffPacket({ + handoff: lastHandoff, + history: input.history, + request, + response, + sourceChain: nextChain, + state: { + source: skill.name, + }, + }); + + request = { + ...request, + context: { + ...(request.context ?? {}), + handoffPacket: packet, + }, + skill: nextSkill.name, + trigger: "handoff", + }; + skill = nextSkill; + chain = nextChain; + } + + const finalHandoff: HandoffRequest = { + confidence: 0.4, + evidenceRefs: [], + expectedOutput: "text", + reason: "max-hop", + risk: "medium", + targetAgent: "supervisor", + taskSpec: "Eval dispatch exceeded max loop depth.", + }; + const fallbackChain = chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"]; + return { + chain: fallbackChain, + finalSkill: "supervisor", + response: withHandoffMetadata( + { + confidence: 0.4, + requestId: input.request.requestId, + source: "template", + text: "확인이 필요한 내용이라 운영자에게 전달했어요.", + }, + fallbackChain, + fallbackChain.length - 1, + finalHandoff, + ), + }; +} diff --git a/src/eval/judges/handoff-judge.test.ts b/src/eval/judges/handoff-judge.test.ts new file mode 100644 index 0000000..af833d1 --- /dev/null +++ b/src/eval/judges/handoff-judge.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; + +import type { AgentResponse } from "@agent/types"; +import { judgeHandoff } from "./handoff-judge.js"; + +const response: AgentResponse = { + confidence: 0.8, + metadata: { + handoffChain: ["onboarding", "moderation", "supervisor"], + handoffLastRequest: { + confidence: 0.6, + evidenceRefs: [], + expectedOutput: "text", + reason: "same-agent-revisit", + risk: "medium", + targetAgent: "supervisor", + taskSpec: "loop blocked", + }, + }, + requestId: "request-1", + source: "template", + text: "sent to review", +}; + +describe("judgeHandoff", () => { + it("passes exact chain, final skill, target, and reason checks", () => { + expect( + judgeHandoff( + { + expectedFinalSkill: "supervisor", + expectedHandoff: { + expectedReason: "same-agent-revisit", + expectedTarget: "supervisor", + shouldHandoff: true, + }, + expectedSourceChain: { + chain: ["onboarding", "moderation", "supervisor"], + mode: "exact", + }, + }, + response, + ), + ).toMatchObject({ + actualChain: ["onboarding", "moderation", "supervisor"], + actualFinalSkill: "supervisor", + actualHandoffReason: "same-agent-revisit", + actualHandoffTarget: "supervisor", + passed: true, + }); + }); + + it("supports prefix chain checks", () => { + expect( + judgeHandoff( + { + expectedSourceChain: { + chain: ["onboarding", "moderation"], + mode: "prefix", + }, + }, + response, + ).passed, + ).toBe(true); + }); + + it("supports contains chain checks", () => { + expect( + judgeHandoff( + { + expectedSourceChain: { + chain: ["moderation"], + mode: "contains", + }, + }, + response, + ).passed, + ).toBe(true); + }); + + it("fails when required metadata is missing", () => { + const result = judgeHandoff( + { + expectedFinalSkill: "supervisor", + expectedHandoff: { shouldHandoff: true }, + expectedSourceChain: { + chain: ["supervisor"], + mode: "contains", + }, + }, + { ...response, metadata: {} }, + ); + + expect(result.passed).toBe(false); + expect(result.checks.filter((check) => !check.passed)).toHaveLength(3); + }); + + it("skips unspecified checks", () => { + const result = judgeHandoff({}, response); + expect(result.passed).toBe(true); + expect(result.checks.every((check) => check.skipped)).toBe(true); + }); +}); diff --git a/src/eval/judges/handoff-judge.ts b/src/eval/judges/handoff-judge.ts new file mode 100644 index 0000000..401a534 --- /dev/null +++ b/src/eval/judges/handoff-judge.ts @@ -0,0 +1,152 @@ +import type { AgentResponse, HandoffRequest } from "@agent/types"; +import type { + EvalCaseExpected, + HandoffJudgeCheck, + HandoffJudgeResult, +} from "../types.js"; + +function asStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.filter((item): item is string => typeof item === "string"); +} + +function readHandoffRequest( + value: unknown, +): Pick | null { + if (typeof value !== "object" || value === null) { + return null; + } + const record = value as Partial; + if (typeof record.reason !== "string" || typeof record.targetAgent !== "string") { + return null; + } + return { + reason: record.reason, + targetAgent: record.targetAgent, + }; +} + +function chainMatches( + actual: readonly string[], + expected: readonly string[], + mode: "contains" | "exact" | "prefix", +): boolean { + if (mode === "exact") { + return ( + actual.length === expected.length && + expected.every((skill, index) => actual[index] === skill) + ); + } + if (mode === "prefix") { + return expected.every((skill, index) => actual[index] === skill); + } + + if (expected.length === 0) { + return true; + } + return actual.some((_skill, startIndex) => + expected.every((skill, offset) => actual[startIndex + offset] === skill), + ); +} + +function skippedCheck(name: HandoffJudgeCheck["name"]): HandoffJudgeCheck { + return { + name, + passed: true, + skipped: true, + }; +} + +export function judgeHandoff( + expected: EvalCaseExpected, + response: AgentResponse, + finalSkill?: string, +): HandoffJudgeResult { + const metadata = response.metadata ?? {}; + const actualChain = asStringArray(metadata["handoffChain"]); + const finalSkillFromChain = actualChain.at(-1); + const actualFinalSkill = + finalSkill ?? (typeof finalSkillFromChain === "string" ? finalSkillFromChain : ""); + const handoff = + readHandoffRequest(metadata["handoffRequest"]) ?? + readHandoffRequest(metadata["handoffLastRequest"]); + const actualHandoffReason = handoff?.reason; + const actualHandoffTarget = handoff?.targetAgent; + const checks: HandoffJudgeCheck[] = []; + + if (expected.expectedHandoff?.shouldHandoff === undefined) { + checks.push(skippedCheck("should-handoff")); + } else { + const didHandoff = actualChain.length > 1 || Boolean(handoff); + const passed = didHandoff === expected.expectedHandoff.shouldHandoff; + checks.push({ + detail: `expected=${expected.expectedHandoff.shouldHandoff} actual=${didHandoff}`, + name: "should-handoff", + passed, + skipped: false, + }); + } + + if (!expected.expectedHandoff?.expectedReason) { + checks.push(skippedCheck("handoff-reason")); + } else { + const passed = actualHandoffReason === expected.expectedHandoff.expectedReason; + checks.push({ + detail: `expected=${expected.expectedHandoff.expectedReason} actual=${actualHandoffReason ?? "-"}`, + name: "handoff-reason", + passed, + skipped: false, + }); + } + + if (!expected.expectedHandoff?.expectedTarget) { + checks.push(skippedCheck("handoff-target")); + } else { + const passed = actualHandoffTarget === expected.expectedHandoff.expectedTarget; + checks.push({ + detail: `expected=${expected.expectedHandoff.expectedTarget} actual=${actualHandoffTarget ?? "-"}`, + name: "handoff-target", + passed, + skipped: false, + }); + } + + if (!expected.expectedFinalSkill) { + checks.push(skippedCheck("final-skill")); + } else { + const passed = actualFinalSkill === expected.expectedFinalSkill; + checks.push({ + detail: `expected=${expected.expectedFinalSkill} actual=${actualFinalSkill || "-"}`, + name: "final-skill", + passed, + skipped: false, + }); + } + + if (!expected.expectedSourceChain) { + checks.push(skippedCheck("chain")); + } else { + const passed = chainMatches( + actualChain, + expected.expectedSourceChain.chain, + expected.expectedSourceChain.mode, + ); + checks.push({ + detail: `mode=${expected.expectedSourceChain.mode} expected=${expected.expectedSourceChain.chain.join(">")} actual=${actualChain.join(">") || "-"}`, + name: "chain", + passed, + skipped: false, + }); + } + + return { + actualChain, + actualFinalSkill, + ...(actualHandoffReason ? { actualHandoffReason } : {}), + ...(actualHandoffTarget ? { actualHandoffTarget } : {}), + checks, + passed: checks.every((check) => check.skipped || check.passed), + }; +} diff --git a/src/eval/review/cli.ts b/src/eval/review/cli.ts index cec7091..c0e7fc8 100644 --- a/src/eval/review/cli.ts +++ b/src/eval/review/cli.ts @@ -25,7 +25,7 @@ export interface ReviewCliOptions { resetPending?: boolean | undefined; } -function getReviewerId(): string { +export function getReviewerId(): string { const configured = process.env["REVIEWER_ID"]?.trim(); if (configured) { return configured; diff --git a/src/eval/runner.ts b/src/eval/runner.ts index c073515..9db9fc8 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -2,8 +2,8 @@ import { existsSync } from "node:fs"; import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import { basename, dirname, resolve } from "node:path"; -import type { AgentRequest } from "@agent/types"; -import { GarnetClient } from "@common/garnet"; +import type { AgentRequest, AgentResponse, HandoffReason } from "@agent/types"; +import { GarnetClient, type StreamEntry } from "@common/garnet"; import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects"; import { NatsClient } from "@common/nats"; import { getConversationSummaryKey } from "@conv/conversation-store"; @@ -17,6 +17,7 @@ import type { Skill } from "@skills/skill"; import { ModerationSkill } from "@skills/moderation/index"; import { classifyModerationIntent } from "@skills/moderation/intents"; import { OnboardingSkill } from "@skills/onboarding/index"; +import { SupervisorSkill } from "@skills/supervisor/index"; import { ProcessTraceCollector, runWithProcessTrace, @@ -30,6 +31,8 @@ import type { } from "@skills/onboarding/state/onboarding-state"; import { noopShadowEvalPublisher } from "./shadow-bus.js"; import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js"; +import { runEvalDispatch } from "./handoff-dispatch.js"; +import { judgeHandoff } from "./judges/handoff-judge.js"; import { judgeSkillWithLlm } from "./judges/llm-judge.js"; import { judgeProcessRule } from "./judges/process-judge.js"; import { judgeRuleCase } from "./judges/rule-judge.js"; @@ -72,11 +75,17 @@ const SKILL_RUNNER_FACTORIES: Record Skill> = deps.room, noopShadowEvalPublisher, ), + supervisor: (deps) => + new SupervisorSkill(deps.garnet, { + publish: (_subject: string, _payload: T): Promise => + Promise.resolve(), + }), }; class InMemoryGarnetClient extends GarnetClient { private readonly hashStore = new Map>(); private readonly stringStore = new Map(); + private readonly streamStore = new Map(); getJson(key: string): Promise { const raw = this.stringStore.get(key); @@ -105,6 +114,18 @@ class InMemoryGarnetClient extends GarnetClient { return Promise.resolve(); } + setStringIfAbsent( + key: string, + value: string, + _ttlSeconds?: number, + ): Promise { + if (this.stringStore.has(key)) { + return Promise.resolve(false); + } + this.stringStore.set(key, value); + return Promise.resolve(true); + } + delete(key: string): Promise { const stringDeleted = this.stringStore.delete(key); const hashDeleted = this.hashStore.delete(key); @@ -122,6 +143,39 @@ class InMemoryGarnetClient extends GarnetClient { this.hashStore.set(key, { ...fields }); return Promise.resolve(); } + + expireKey(_key: string, _ttlSeconds: number): Promise { + return Promise.resolve(true); + } + + appendStream( + key: string, + fields: Record, + _options?: { maxLen?: number | undefined }, + ): Promise { + const entries = this.streamStore.get(key) ?? []; + const id = `${entries.length + 1}-0`; + entries.push({ + id, + message: { ...fields }, + }); + this.streamStore.set(key, entries); + return Promise.resolve(id); + } + + readStreamRange( + key: string, + start: string, + end: string, + options?: { count?: number | undefined }, + ): Promise { + const entries = (this.streamStore.get(key) ?? []).filter((entry) => { + const afterStart = start === "-" || entry.id >= start; + const beforeEnd = end === "+" || entry.id <= end; + return afterStart && beforeEnd; + }); + return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries); + } } class InMemoryRoomContextService extends RoomContextService { @@ -277,6 +331,19 @@ function getStringArrayField( return strings.length > 0 ? strings : []; } +function isHandoffReason(value: string | undefined): value is HandoffReason { + return ( + value === "out-of-scope" || + value === "missing-tool" || + value === "low-confidence" || + value === "validation-failed" || + value === "same-agent-revisit" || + value === "max-hop" || + value === "high-risk-action" || + value === "policy-risk" + ); +} + function toAgentRequest(raw: Record): AgentRequest { const requestId = getStringField(raw, "requestId"); const skill = getStringField(raw, "skill"); @@ -360,10 +427,14 @@ function parseExpected(raw: Record | undefined): EvalCaseExpect const responseMode = getStringField(raw, "responseMode"); const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny"); const mustNotInclude = getStringArrayField(raw, "mustNotInclude"); + const expectedFinalSkill = getStringField(raw, "expectedFinalSkill"); if (intent) { expected.intent = intent; } + if (expectedFinalSkill) { + expected.expectedFinalSkill = expectedFinalSkill; + } if (responseMode) { expected.responseMode = responseMode; } @@ -409,6 +480,42 @@ function parseExpected(raw: Record | undefined): EvalCaseExpect } } + const expectedHandoffRaw = raw["expectedHandoff"]; + if (isRecord(expectedHandoffRaw)) { + const expectedReason = getStringField(expectedHandoffRaw, "expectedReason"); + const expectedTarget = getStringField(expectedHandoffRaw, "expectedTarget"); + const shouldHandoff = expectedHandoffRaw["shouldHandoff"]; + const expectedHandoff: NonNullable = {}; + + if (typeof shouldHandoff === "boolean") { + expectedHandoff.shouldHandoff = shouldHandoff; + } + if (isHandoffReason(expectedReason)) { + expectedHandoff.expectedReason = expectedReason; + } + if (expectedTarget) { + expectedHandoff.expectedTarget = expectedTarget; + } + if (Object.keys(expectedHandoff).length > 0) { + expected.expectedHandoff = expectedHandoff; + } + } + + const expectedSourceChainRaw = raw["expectedSourceChain"]; + if (isRecord(expectedSourceChainRaw)) { + const mode = getStringField(expectedSourceChainRaw, "mode"); + const chain = getStringArrayField(expectedSourceChainRaw, "chain"); + if ( + chain && + (mode === "exact" || mode === "prefix" || mode === "contains") + ) { + expected.expectedSourceChain = { + chain, + mode, + }; + } + } + return expected; } @@ -538,6 +645,7 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary { const llmResults = results.flatMap((result) => result.llmJudge.skipped ? [] : [result.llmJudge], ); + const passedResults = results.filter(isCasePassed); return { averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)), @@ -563,16 +671,19 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary { results.length === 0 ? 0 : round( - results.filter((result) => result.ruleJudge.passed).length / - results.length, + passedResults.length / results.length, ), - passedCases: results.filter((result) => result.ruleJudge.passed).length, + passedCases: passedResults.length, ruleAverageScore: averageOrZero( results.map((result) => result.ruleJudge.score), ), }; } +function isCasePassed(result: EvalCaseResult): boolean { + return result.ruleJudge.passed && (result.handoffJudge?.passed ?? true); +} + function createBaselineDelta( current: EvalReportSummary, baseline: EvalReportSummary, @@ -717,16 +828,27 @@ async function evaluateCase( input.roomContext && input.roomContext.length > 0 ? new InMemoryRoomContextService(toRoomMessages(input.roomContext)) : undefined; + const deps = { + garnet: inMemoryGarnet, + llm: llmClient, + room: roomContextService, + }; + if (skillName === "handoff") { + return evaluateHandoffCase( + evalCase, + input, + Object.values(SKILL_RUNNER_FACTORIES).map((factory) => factory(deps)), + llmClient, + enableLlmJudge, + ); + } + const effectiveSkill = input.request.skill || skillName; const factory = SKILL_RUNNER_FACTORIES[effectiveSkill]; if (!factory) { throw new Error(`No eval runner factory for skill: ${effectiveSkill}`); } - const skill = factory({ - garnet: inMemoryGarnet, - llm: llmClient, - room: roomContextService, - }); + const skill = factory(deps); return runWithContext( { @@ -874,6 +996,105 @@ async function evaluateOnboardingCase( }); } +async function evaluateHandoffCase( + evalCase: EvalCaseDefinition, + input: EvalCaseInput, + skills: readonly Skill[], + _llmClient: LlmClient, + _enableLlmJudge: boolean, +): Promise { + const startedAt = Date.now(); + const derived: EvalDerivedSnapshot = { + allowShortResponse: true, + intent: "handoff", + phase: input.request.skill, + responseMode: "template", + shouldRespond: true, + suggestedMaxTokens: 192, + tier: 0, + }; + + try { + const result = await runEvalDispatch({ + history: input.conversationHistory ?? [], + request: input.request, + skills, + }); + const latencyMs = Date.now() - startedAt; + const handoffJudge = judgeHandoff( + evalCase.expected, + result.response, + result.finalSkill, + ); + + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: { + confidence: 1, + intent: "handoff", + source: "regex", + }, + derived: { + ...derived, + phase: result.finalSkill, + }, + request: input.request, + response: result.response, + ruleJudge: judgeRuleCase(evalCase.expected, result.response.text, { + derived, + intent: "handoff", + responseMode: "template", + }), + llmJudge: { + skipped: true, + reason: "handoff-eval", + }, + handoffJudge, + }; + } catch (error) { + const latencyMs = Date.now() - startedAt; + const errorMessage = error instanceof Error ? error.message : String(error); + const response: AgentResponse = { + confidence: 0, + metadata: { + error: errorMessage, + }, + requestId: input.request.requestId, + source: "template", + text: "", + }; + + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: { + confidence: 0, + intent: "handoff", + source: "regex", + }, + derived, + request: input.request, + response, + ruleJudge: judgeRuleCase(evalCase.expected, "", { + derived, + intent: "handoff", + responseMode: "template", + }), + llmJudge: { + skipped: true, + reason: "handoff-error", + }, + handoffJudge: judgeHandoff(evalCase.expected, response, ""), + error: errorMessage, + }; + } +} + async function evaluateGenericSkillCase( evalCase: EvalCaseDefinition, input: EvalCaseInput, diff --git a/src/eval/types.ts b/src/eval/types.ts index a1b2456..1f774fa 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -2,6 +2,7 @@ import type { AgentRequest, AgentResponse, ConversationMessage, + HandoffReason, } from "@agent/types"; import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state"; @@ -30,6 +31,16 @@ export interface EvalCaseInput { } export interface EvalCaseExpected { + expectedFinalSkill?: string | undefined; + expectedHandoff?: { + expectedReason?: HandoffReason | undefined; + expectedTarget?: string | undefined; + shouldHandoff?: boolean | undefined; + } | undefined; + expectedSourceChain?: { + chain: string[]; + mode: "contains" | "exact" | "prefix"; + } | undefined; intent?: string | undefined; mustIncludeAny?: string[] | undefined; mustNotInclude?: string[] | undefined; @@ -121,6 +132,27 @@ export interface ProcessJudgeResult { totalCount: number; } +export interface HandoffJudgeCheck { + detail?: string | undefined; + name: + | "chain" + | "final-skill" + | "handoff-reason" + | "handoff-target" + | "should-handoff"; + passed: boolean; + skipped: boolean; +} + +export interface HandoffJudgeResult { + actualChain: string[]; + actualFinalSkill: string; + actualHandoffReason?: HandoffReason | undefined; + actualHandoffTarget?: string | undefined; + checks: HandoffJudgeCheck[]; + passed: boolean; +} + export interface EvalDerivedSnapshot { abstentionScore?: number | undefined; allowShortResponse: boolean; @@ -150,6 +182,7 @@ export interface EvalCaseResult { response: AgentResponse; ruleJudge: RuleJudgeResult; llmJudge: LlmJudgeResult; + handoffJudge?: HandoffJudgeResult | undefined; processJudge?: ProcessJudgeResult | undefined; processTrace?: ProcessTraceSnapshot | undefined; error?: string | undefined; diff --git a/src/operations/cli.test.ts b/src/operations/cli.test.ts new file mode 100644 index 0000000..e4c42de --- /dev/null +++ b/src/operations/cli.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; + +import { + formatReviewList, + formatReviewShow, + formatReviewStats, +} from "./cli.js"; + +describe("operations review CLI formatters", () => { + it("formats review list rows as tab-separated output", () => { + expect( + formatReviewList([ + { + chatroomId: "room-1", + id: "1-0", + queuedAt: "2026-05-18T00:00:00.000Z", + reason: "low-confidence", + requestId: "request-1", + target: "human-review", + userId: "user-1", + }, + ]), + ).toContain("1-0\t2026-05-18T00:00:00.000Z\tlow-confidence"); + }); + + it("formats packet details for show output", () => { + expect( + formatReviewShow({ + chatroomId: "room-1", + id: "1-0", + packet: { + constraints: [], + currentState: {}, + evidence: [ + { + id: "e1", + kind: "message", + summary: "unclear message", + }, + ], + expectedOutputSchema: "text", + handoffReason: "low-confidence", + sourceChain: ["onboarding", "supervisor"], + summary: "needs review", + userGoal: "의도 확인", + }, + queuedAt: "2026-05-18T00:00:00.000Z", + reason: "low-confidence", + requestId: "request-1", + target: "human-review", + userId: "user-1", + }), + ).toContain("sourceChain: onboarding > supervisor"); + }); + + it("formats aggregate stats", () => { + expect( + formatReviewStats({ + averagePendingAgeMs: 60_000, + byReason: { + "low-confidence": { + pending: 1, + resolved: 2, + total: 3, + }, + }, + pending: 1, + resolved: 2, + topSourceChains: [{ chain: "onboarding>supervisor", count: 3 }], + total: 3, + }), + ).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2"); + }); +}); diff --git a/src/operations/cli.ts b/src/operations/cli.ts new file mode 100644 index 0000000..b5e9a74 --- /dev/null +++ b/src/operations/cli.ts @@ -0,0 +1,299 @@ +import { pathToFileURL } from "node:url"; + +import type { HandoffReason } from "@agent/types"; +import { loadEnvironmentFromNodeEnv } from "@common/env-loader"; +import { connectGarnetForCli } from "@common/garnet-cli"; +import { initLogger } from "@obs/logger"; +import { + getSupervisorReview, + getSupervisorReviewStats, + listSupervisorReview, + resolveSupervisorReview, + type SupervisorReviewAction, + type SupervisorReviewItem, + type SupervisorReviewStats, +} from "@skills/supervisor/review-queue"; +import { + getFlagString, + hasFlag, + parseArgs, + parseDurationMs, + parsePositiveInteger, +} from "../eval/cli-args.js"; +import { getReviewerId } from "../eval/review/cli.js"; + +function isHandoffReason(value: string | undefined): value is HandoffReason { + return ( + value === "out-of-scope" || + value === "missing-tool" || + value === "low-confidence" || + value === "validation-failed" || + value === "same-agent-revisit" || + value === "max-hop" || + value === "high-risk-action" || + value === "policy-risk" + ); +} + +function isSupervisorReviewAction( + value: string | undefined, +): value is SupervisorReviewAction { + return value === "manual-reply" || value === "reroute" || value === "dismiss"; +} + +function sinceDurationToIso(value: string | undefined): string | undefined { + if (!value?.trim()) { + return undefined; + } + return new Date(Date.now() - parseDurationMs(value)).toISOString(); +} + +function stringifyOptional(value: string | undefined): string { + return value?.trim() ? value : "-"; +} + +function formatAge(ms: number): string { + if (!Number.isFinite(ms) || ms <= 0) { + return "0s"; + } + const seconds = Math.round(ms / 1000); + if (seconds < 60) { + return `${seconds}s`; + } + const minutes = Math.round(seconds / 60); + if (minutes < 60) { + return `${minutes}m`; + } + const hours = Math.round(minutes / 60); + if (hours < 48) { + return `${hours}h`; + } + return `${Math.round(hours / 24)}d`; +} + +export function formatReviewList(items: SupervisorReviewItem[]): string { + const rows = [ + [ + "id", + "queuedAt", + "reason", + "chatroomId", + "requestId", + "target", + "resolved", + ], + ...items.map((item) => [ + item.id, + item.queuedAt, + item.reason, + item.chatroomId, + item.requestId, + item.target, + item.resolution ? "yes" : "no", + ]), + ]; + return rows.map((row) => row.join("\t")).join("\n"); +} + +export function formatReviewShow(item: SupervisorReviewItem): string { + const packet = item.packet; + const evidence = + packet?.evidence + .map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`) + .join("\n") || "-"; + + return [ + `id: ${item.id}`, + `queuedAt: ${item.queuedAt}`, + `reason: ${item.reason}`, + `target: ${item.target}`, + `chatroomId: ${item.chatroomId}`, + `userId: ${item.userId}`, + `requestId: ${item.requestId}`, + `sourceChain: ${packet?.sourceChain.join(" > ") || "-"}`, + `userGoal: ${stringifyOptional(packet?.userGoal)}`, + `summary: ${stringifyOptional(packet?.summary)}`, + "[evidence]", + evidence, + item.resolution + ? [ + "[resolution]", + `action: ${item.resolution.action}`, + `resolvedAt: ${item.resolution.resolvedAt}`, + `resolvedBy: ${item.resolution.resolvedBy}`, + `note: ${stringifyOptional(item.resolution.note)}`, + ].join("\n") + : "[resolution]\n-", + ].join("\n"); +} + +export function formatReviewStats(stats: SupervisorReviewStats): string { + const reasonRows = Object.entries(stats.byReason) + .sort(([left], [right]) => left.localeCompare(right)) + .map( + ([reason, counts]) => + `${reason}\ttotal=${counts.total}\tpending=${counts.pending}\tresolved=${counts.resolved}`, + ); + const sourceRows = stats.topSourceChains.map( + (item) => `${item.chain}\tcount=${item.count}`, + ); + + return [ + `total=${stats.total} pending=${stats.pending} resolved=${stats.resolved} averagePendingAge=${formatAge(stats.averagePendingAgeMs)}`, + "[byReason]", + ...(reasonRows.length > 0 ? reasonRows : ["-"]), + "[topSourceChains]", + ...(sourceRows.length > 0 ? sourceRows : ["-"]), + ].join("\n"); +} + +async function handleReviewList( + flags: Map, +): Promise { + const reasonRaw = getFlagString(flags, "reason"); + if (reasonRaw && !isHandoffReason(reasonRaw)) { + throw new Error(`Invalid --reason: ${reasonRaw}`); + } + const reason = isHandoffReason(reasonRaw) ? reasonRaw : undefined; + + const connection = await connectGarnetForCli(); + try { + const items = await listSupervisorReview(connection.garnetClient, { + includeResolved: hasFlag(flags, "include-resolved"), + limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? 50, + reason, + sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since")), + untilQueuedAt: getFlagString(flags, "until"), + }); + console.info( + hasFlag(flags, "json") + ? JSON.stringify(items, null, 2) + : formatReviewList(items), + ); + } finally { + await connection.close(); + } +} + +async function handleReviewShow( + id: string | undefined, + flags: Map, +): Promise { + if (!id?.trim()) { + throw new Error("review:show requires an entry id"); + } + + const connection = await connectGarnetForCli(); + try { + const item = await getSupervisorReview(connection.garnetClient, id); + if (!item) { + process.exitCode = 2; + console.error(`[review] entry not found: ${id}`); + return; + } + console.info( + hasFlag(flags, "json") + ? JSON.stringify(item, null, 2) + : formatReviewShow(item), + ); + } finally { + await connection.close(); + } +} + +async function handleReviewResolve( + id: string | undefined, + flags: Map, +): Promise { + if (!id?.trim()) { + throw new Error("review:resolve requires an entry id"); + } + const action = getFlagString(flags, "action"); + if (!isSupervisorReviewAction(action)) { + throw new Error( + "--action is required and must be manual-reply, reroute, or dismiss", + ); + } + + const connection = await connectGarnetForCli(); + try { + const item = await getSupervisorReview(connection.garnetClient, id); + if (!item) { + process.exitCode = 2; + console.error(`[review] entry not found: ${id}`); + return; + } + + await resolveSupervisorReview(connection.garnetClient, id, { + action, + resolvedAt: new Date().toISOString(), + resolvedBy: getFlagString(flags, "by") ?? getReviewerId(), + ...(getFlagString(flags, "note") + ? { note: getFlagString(flags, "note") } + : {}), + }); + console.info(`[review] resolved id=${id} action=${action}`); + } finally { + await connection.close(); + } +} + +async function handleReviewStats( + flags: Map, +): Promise { + const connection = await connectGarnetForCli(); + try { + const stats = await getSupervisorReviewStats(connection.garnetClient, { + limit: parsePositiveInteger(getFlagString(flags, "limit")), + sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since") ?? "7d"), + }); + console.info( + hasFlag(flags, "json") + ? JSON.stringify(stats, null, 2) + : formatReviewStats(stats), + ); + } finally { + await connection.close(); + } +} + +async function main(): Promise { + loadEnvironmentFromNodeEnv(); + initLogger(); + + const { command, flags, positionals } = parseArgs(process.argv.slice(2)); + if (command === "review:list") { + await handleReviewList(flags); + return; + } + if (command === "review:show") { + await handleReviewShow(positionals[0], flags); + return; + } + if (command === "review:resolve") { + await handleReviewResolve(positionals[0], flags); + return; + } + if (command === "review:stats") { + await handleReviewStats(flags); + return; + } + + throw new Error( + "Unknown command. Use review:list, review:show, review:resolve, or review:stats.", + ); +} + +function isDirectCliEntry(): boolean { + const entrypoint = process.argv[1]; + return entrypoint + ? import.meta.url === pathToFileURL(entrypoint).href + : false; +} + +if (isDirectCliEntry()) { + main().catch((error) => { + process.exitCode = process.exitCode || 1; + console.error(error instanceof Error ? error.message : String(error)); + }); +} diff --git a/src/skills/supervisor/review-queue.test.ts b/src/skills/supervisor/review-queue.test.ts new file mode 100644 index 0000000..0bb238e --- /dev/null +++ b/src/skills/supervisor/review-queue.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from "vitest"; + +import type { StreamEntry } from "@common/garnet"; +import { + getSupervisorReview, + getSupervisorReviewStats, + listSupervisorReview, + parseSupervisorReviewEntry, + resolveSupervisorReview, + SUPERVISOR_RESOLVED_HASH_PREFIX, + SUPERVISOR_REVIEW_STREAM_KEY, +} from "./review-queue.js"; + +class FakeGarnet { + readonly hashes = new Map>(); + readonly stream: StreamEntry[]; + + constructor(stream: StreamEntry[]) { + this.stream = stream; + } + + readStreamRange( + key: string, + start: string, + end: string, + options?: { count?: number | undefined }, + ): Promise { + expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY); + const entries = this.stream.filter((entry) => { + const afterStart = start === "-" || entry.id >= start; + const beforeEnd = end === "+" || entry.id <= end; + return afterStart && beforeEnd; + }); + return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries); + } + + getHash(key: string): Promise> { + return Promise.resolve({ ...(this.hashes.get(key) ?? {}) }); + } + + setHash(key: string, fields: Record): Promise { + this.hashes.set(key, { ...fields }); + return Promise.resolve(); + } + + expireKey(): Promise { + return Promise.resolve(true); + } +} + +const packet = { + constraints: [], + currentState: {}, + evidence: [], + expectedOutputSchema: "text", + handoffReason: "out-of-scope", + sourceChain: ["onboarding", "moderation"], + summary: "user asked about ads", + userGoal: "광고 규칙 문의", +}; + +const entries: StreamEntry[] = [ + { + id: "1-0", + message: { + chatroomId: "room-1", + packet: JSON.stringify(packet), + queuedAt: "2026-05-18T00:00:00.000Z", + reason: "out-of-scope", + requestId: "request-1", + target: "moderation", + type: "supervisor-handoff", + userId: "user-1", + }, + }, + { + id: "2-0", + message: { + chatroomId: "room-2", + queuedAt: "2026-05-18T01:00:00.000Z", + reason: "low-confidence", + requestId: "request-2", + target: "human-review", + type: "supervisor-handoff", + userId: "user-2", + }, + }, +]; + +describe("supervisor review queue", () => { + const firstEntry = entries[0]!; + + it("parses a stream entry with the preserved handoff packet", () => { + expect(parseSupervisorReviewEntry(firstEntry)).toMatchObject({ + chatroomId: "room-1", + id: "1-0", + packet, + reason: "out-of-scope", + target: "moderation", + }); + }); + + it("lists pending entries and filters resolved items by default", async () => { + const garnet = new FakeGarnet(entries); + await resolveSupervisorReview(garnet, "1-0", { + action: "manual-reply", + resolvedAt: "2026-05-18T02:00:00.000Z", + resolvedBy: "operator@example.com", + }); + + await expect(listSupervisorReview(garnet)).resolves.toHaveLength(1); + await expect( + listSupervisorReview(garnet, { includeResolved: true, reason: "out-of-scope" }), + ).resolves.toMatchObject([ + { + id: "1-0", + resolution: { + action: "manual-reply", + resolvedBy: "operator@example.com", + }, + }, + ]); + }); + + it("gets a single entry with resolution metadata", async () => { + const garnet = new FakeGarnet(entries); + garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, { + action: "dismiss", + note: "duplicate", + resolvedAt: "2026-05-18T02:00:00.000Z", + resolvedBy: "operator@example.com", + }); + + await expect(getSupervisorReview(garnet, "1-0")).resolves.toMatchObject({ + id: "1-0", + resolution: { + action: "dismiss", + note: "duplicate", + }, + }); + }); + + it("computes reason totals and top source chains", async () => { + const garnet = new FakeGarnet(entries); + garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, { + action: "reroute", + resolvedAt: "2026-05-18T02:00:00.000Z", + resolvedBy: "operator@example.com", + }); + + await expect(getSupervisorReviewStats(garnet)).resolves.toMatchObject({ + byReason: { + "low-confidence": { + pending: 1, + resolved: 0, + total: 1, + }, + "out-of-scope": { + pending: 0, + resolved: 1, + total: 1, + }, + }, + pending: 1, + resolved: 1, + topSourceChains: [{ chain: "onboarding>moderation", count: 1 }], + total: 2, + }); + }); +}); diff --git a/src/skills/supervisor/review-queue.ts b/src/skills/supervisor/review-queue.ts index 92c2ca1..088567e 100644 --- a/src/skills/supervisor/review-queue.ts +++ b/src/skills/supervisor/review-queue.ts @@ -1,14 +1,344 @@ -import type { GarnetClient } from "@common/garnet"; +import type { HandoffPacket } from "@agent/handoff-packet"; +import type { HandoffReason } from "@agent/types"; +import type { GarnetClient, StreamEntry } from "@common/garnet"; +import { + getSupervisorReviewQueueMaxLen, + getSupervisorReviewResolutionTtlSec, +} from "../../config.js"; -const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue"; +export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue"; +export const SUPERVISOR_RESOLVED_HASH_PREFIX = + "lolympus-agent:review-resolved:"; + +export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss"; + +export interface SupervisorReviewEntry { + id: string; + queuedAt: string; + chatroomId: string; + userId: string; + requestId: string; + reason: HandoffReason; + target: string; + packet?: HandoffPacket | undefined; + extra?: Record | undefined; +} + +export interface SupervisorReviewResolution { + resolvedAt: string; + resolvedBy: string; + action: SupervisorReviewAction; + note?: string | undefined; +} + +export interface SupervisorReviewItem extends SupervisorReviewEntry { + resolution?: SupervisorReviewResolution | undefined; +} + +export interface ListSupervisorReviewOptions { + includeResolved?: boolean | undefined; + limit?: number | undefined; + reason?: HandoffReason | undefined; + since?: string | undefined; + sinceQueuedAt?: string | undefined; + until?: string | undefined; + untilQueuedAt?: string | undefined; +} + +export interface SupervisorReviewReasonStats { + pending: number; + resolved: number; + total: number; +} + +export interface SupervisorReviewStats { + averagePendingAgeMs: number; + byReason: Record; + pending: number; + resolved: number; + topSourceChains: Array<{ chain: string; count: number }>; + total: number; +} + +type SupervisorReviewListGarnetClient = Pick< + GarnetClient, + "getHash" | "readStreamRange" +>; + +type SupervisorReviewResolveGarnetClient = Pick< + GarnetClient, + "expireKey" | "setHash" +>; + +const BASE_FIELDS = new Set([ + "chatroomId", + "packet", + "queuedAt", + "reason", + "requestId", + "target", + "type", + "userId", +]); + +function isHandoffReason(value: string | undefined): value is HandoffReason { + return ( + value === "out-of-scope" || + value === "missing-tool" || + value === "low-confidence" || + value === "validation-failed" || + value === "same-agent-revisit" || + value === "max-hop" || + value === "high-risk-action" || + value === "policy-risk" + ); +} + +function isSupervisorReviewAction( + value: string | undefined, +): value is SupervisorReviewAction { + return value === "manual-reply" || value === "reroute" || value === "dismiss"; +} + +function parseOptionalPacket(value: string | undefined): HandoffPacket | undefined { + if (!value?.trim() || value.trim() === "null") { + return undefined; + } + + try { + const parsed = JSON.parse(value) as unknown; + return typeof parsed === "object" && parsed !== null + ? (parsed as HandoffPacket) + : undefined; + } catch { + return undefined; + } +} + +function parseResolution( + fields: Record, +): SupervisorReviewResolution | undefined { + if ( + !fields["resolvedAt"] || + !fields["resolvedBy"] || + !isSupervisorReviewAction(fields["action"]) + ) { + return undefined; + } + + return { + action: fields["action"], + resolvedAt: fields["resolvedAt"], + resolvedBy: fields["resolvedBy"], + ...(fields["note"] ? { note: fields["note"] } : {}), + }; +} + +function resolutionKey(id: string): string { + return `${SUPERVISOR_RESOLVED_HASH_PREFIX}${id}`; +} + +function isAfterOrEqual(value: string, lowerBound: string | undefined): boolean { + return !lowerBound || value >= lowerBound; +} + +function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean { + return !upperBound || value <= upperBound; +} export async function appendSupervisorReview( garnetClient: Pick, fields: Record, ): Promise { - await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, { - ...fields, - type: "supervisor-handoff", - queuedAt: new Date().toISOString(), + await garnetClient.appendStream( + SUPERVISOR_REVIEW_STREAM_KEY, + { + ...fields, + type: "supervisor-handoff", + queuedAt: new Date().toISOString(), + }, + { maxLen: getSupervisorReviewQueueMaxLen() }, + ); +} + +export function parseSupervisorReviewEntry( + streamEntry: StreamEntry, +): SupervisorReviewEntry | null { + const { message } = streamEntry; + const reason = message["reason"]; + + if ( + !message["queuedAt"] || + !message["chatroomId"] || + !message["userId"] || + !message["requestId"] || + !message["target"] || + !isHandoffReason(reason) + ) { + return null; + } + + const extra = Object.fromEntries( + Object.entries(message).filter(([key]) => !BASE_FIELDS.has(key)), + ); + const packet = parseOptionalPacket(message["packet"]); + + return { + chatroomId: message["chatroomId"], + id: streamEntry.id, + queuedAt: message["queuedAt"], + reason, + requestId: message["requestId"], + target: message["target"], + userId: message["userId"], + ...(packet ? { packet } : {}), + ...(Object.keys(extra).length > 0 ? { extra } : {}), + }; +} + +export async function getSupervisorReview( + garnetClient: SupervisorReviewListGarnetClient, + id: string, +): Promise { + const entries = await garnetClient.readStreamRange( + SUPERVISOR_REVIEW_STREAM_KEY, + id, + id, + { count: 1 }, + ); + const parsed = entries.flatMap((entry) => { + const review = parseSupervisorReviewEntry(entry); + return review ? [review] : []; + })[0]; + + if (!parsed) { + return null; + } + + const resolution = parseResolution( + await garnetClient.getHash(resolutionKey(parsed.id)), + ); + return { + ...parsed, + ...(resolution ? { resolution } : {}), + }; +} + +export async function listSupervisorReview( + garnetClient: SupervisorReviewListGarnetClient, + options?: ListSupervisorReviewOptions, +): Promise { + const entries = await garnetClient.readStreamRange( + SUPERVISOR_REVIEW_STREAM_KEY, + options?.since ?? "-", + options?.until ?? "+", + ); + const items: SupervisorReviewItem[] = []; + + for (const entry of entries) { + const review = parseSupervisorReviewEntry(entry); + if (!review) { + continue; + } + if (options?.reason && review.reason !== options.reason) { + continue; + } + if (!isAfterOrEqual(review.queuedAt, options?.sinceQueuedAt)) { + continue; + } + if (!isBeforeOrEqual(review.queuedAt, options?.untilQueuedAt)) { + continue; + } + + const resolution = parseResolution( + await garnetClient.getHash(resolutionKey(review.id)), + ); + if (resolution && !options?.includeResolved) { + continue; + } + items.push({ + ...review, + ...(resolution ? { resolution } : {}), + }); + if (options?.limit && items.length >= options.limit) { + break; + } + } + + return items; +} + +export async function resolveSupervisorReview( + garnetClient: SupervisorReviewResolveGarnetClient, + id: string, + resolution: SupervisorReviewResolution, +): Promise { + const fields: Record = { + action: resolution.action, + resolvedAt: resolution.resolvedAt, + resolvedBy: resolution.resolvedBy, + }; + if (resolution.note?.trim()) { + fields.note = resolution.note.trim(); + } + + const key = resolutionKey(id); + await garnetClient.setHash(key, fields); + await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec()); +} + +export async function getSupervisorReviewStats( + garnetClient: SupervisorReviewListGarnetClient, + options?: Omit, +): Promise { + const nowMs = Date.now(); + const items = await listSupervisorReview(garnetClient, { + ...options, + includeResolved: true, }); + const byReason: Record = {}; + const sourceChains = new Map(); + let pendingAgeMs = 0; + let pending = 0; + let resolved = 0; + + for (const item of items) { + const reasonStats = byReason[item.reason] ?? { + pending: 0, + resolved: 0, + total: 0, + }; + reasonStats.total += 1; + + if (item.resolution) { + resolved += 1; + reasonStats.resolved += 1; + } else { + pending += 1; + reasonStats.pending += 1; + const queuedAtMs = Date.parse(item.queuedAt); + if (Number.isFinite(queuedAtMs)) { + pendingAgeMs += Math.max(0, nowMs - queuedAtMs); + } + } + + byReason[item.reason] = reasonStats; + + const chain = item.packet?.sourceChain?.join(">"); + if (chain) { + sourceChains.set(chain, (sourceChains.get(chain) ?? 0) + 1); + } + } + + return { + averagePendingAgeMs: pending === 0 ? 0 : Math.round(pendingAgeMs / pending), + byReason, + pending, + resolved, + topSourceChains: [...sourceChains.entries()] + .map(([chain, count]) => ({ chain, count })) + .sort((left, right) => right.count - left.count) + .slice(0, 5), + total: items.length, + }; } diff --git a/src/skills/supervisor/skill.test.ts b/src/skills/supervisor/skill.test.ts index 0c3c10f..985ada3 100644 --- a/src/skills/supervisor/skill.test.ts +++ b/src/skills/supervisor/skill.test.ts @@ -82,6 +82,15 @@ describe("SupervisorSkill", () => { type: "supervisor-handoff", }, }); + expect(garnet.streams[0]?.fields["packet"]).toEqual(expect.any(String)); + const queuedPacket: unknown = JSON.parse( + String(garnet.streams[0]?.fields["packet"]), + ); + expect(queuedPacket).toMatchObject({ + handoffReason: "max-hop", + sourceChain: ["onboarding", "supervisor"], + userGoal: "가입 상태 확인", + }); expect(nats.published[0]).toMatchObject({ kind: "supervisor-handoff", metric: "agent.supervisor_handoff", diff --git a/src/skills/supervisor/skill.ts b/src/skills/supervisor/skill.ts index 3b02ed2..57b9ab7 100644 --- a/src/skills/supervisor/skill.ts +++ b/src/skills/supervisor/skill.ts @@ -56,6 +56,7 @@ export class SupervisorSkill implements Skill { await appendSupervisorReview(this.garnetClient, { chatroomId: request.chatroomId, + packet: JSON.stringify(packet ?? null), reason, requestId: request.requestId, target: target ?? "human-review",