Close supervisor review loop and add handoff eval
This commit is contained in:
1 parent
c14adac162
commit
f0b00e5fd2
20 files changed
+2017
-129
No files matched your search
@@ -151,6 +151,10 @@ describe("Agent handoff dispatch", () => {
|
||||
|
||||
expect(nats.responses).toHaveLength(1);
|
||||
expect(nats.responses[0]?.text).toBe("final answer");
|
||||
expect(nats.responses[0]?.metadata).toMatchObject({
|
||||
handoffChain: ["source", "target"],
|
||||
handoffDepth: 1,
|
||||
});
|
||||
expect(store.messages.map((message) => message.content)).toEqual([
|
||||
"final answer",
|
||||
]);
|
||||
@@ -189,6 +193,10 @@ describe("Agent handoff dispatch", () => {
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("handled elsewhere");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
expect(nats.responses[0]?.metadata).toMatchObject({
|
||||
handoffChain: ["source", "target"],
|
||||
handoffDepth: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("routes runtime verifier escalations to supervisor", async () => {
|
||||
@@ -228,6 +236,10 @@ describe("Agent handoff dispatch", () => {
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("sent to review");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
expect(nats.responses[0]?.metadata).toMatchObject({
|
||||
handoffChain: ["moderation", "supervisor"],
|
||||
handoffDepth: 1,
|
||||
});
|
||||
} finally {
|
||||
if (previous === undefined) {
|
||||
delete process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
|
||||
@@ -297,5 +309,9 @@ describe("Agent handoff dispatch", () => {
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("supervisor review");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
expect(nats.responses[0]?.metadata).toMatchObject({
|
||||
handoffChain: ["source", "supervisor"],
|
||||
handoffDepth: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
+24
-4
@@ -282,10 +282,11 @@ export class Agent {
|
||||
response: verified,
|
||||
skill,
|
||||
});
|
||||
const annotated = this.withHandoffMetadata(escalated, chain, depth);
|
||||
|
||||
const handoff = readHandoffRequest(escalated);
|
||||
const handoff = readHandoffRequest(annotated);
|
||||
if (!handoff) {
|
||||
return escalated;
|
||||
return annotated;
|
||||
}
|
||||
|
||||
const next = this.prepareHandoff({
|
||||
@@ -296,11 +297,15 @@ export class Agent {
|
||||
handoff,
|
||||
history,
|
||||
request,
|
||||
response: escalated,
|
||||
response: annotated,
|
||||
});
|
||||
|
||||
if (!next) {
|
||||
return this.makeSupervisorFallback(request, handoff.reason);
|
||||
return this.withHandoffMetadata(
|
||||
this.makeSupervisorFallback(request, handoff.reason),
|
||||
[...chain, "supervisor"],
|
||||
depth + 1,
|
||||
);
|
||||
}
|
||||
|
||||
return this.dispatch(next);
|
||||
@@ -308,6 +313,21 @@ export class Agent {
|
||||
);
|
||||
}
|
||||
|
||||
private withHandoffMetadata(
|
||||
response: AgentResponse,
|
||||
chain: string[],
|
||||
depth: number,
|
||||
): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async runSkillOnce(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { GarnetClient } from "./garnet.js";
|
||||
|
||||
export async function connectGarnetForCli(): Promise<{
|
||||
close: () => Promise<void>;
|
||||
garnetClient: GarnetClient;
|
||||
}> {
|
||||
const garnetClient = new GarnetClient();
|
||||
await garnetClient.connect();
|
||||
return {
|
||||
close: () => garnetClient.close(),
|
||||
garnetClient,
|
||||
};
|
||||
}
|
||||
@@ -218,6 +218,17 @@ export function getEvalReviewDedupeTtlSec(): number {
|
||||
return getPositiveNumber("EVAL_REVIEW_DEDUPE_TTL_SEC", 604_800);
|
||||
}
|
||||
|
||||
export function getSupervisorReviewQueueMaxLen(): number {
|
||||
return getPositiveNumber("SUPERVISOR_REVIEW_QUEUE_MAXLEN", 10_000);
|
||||
}
|
||||
|
||||
export function getSupervisorReviewResolutionTtlSec(): number {
|
||||
return getPositiveNumber(
|
||||
"SUPERVISOR_REVIEW_RESOLUTION_TTL_SEC",
|
||||
90 * 24 * 60 * 60,
|
||||
);
|
||||
}
|
||||
|
||||
export function getEvalCalibrationMinCorr(): number {
|
||||
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
export interface ParsedArgs {
|
||||
command: string;
|
||||
flags: Map<string, string | true>;
|
||||
positionals: string[];
|
||||
}
|
||||
|
||||
export function parsePositiveInteger(
|
||||
value: string | undefined,
|
||||
): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
export function parsePositiveNumber(
|
||||
value: string | undefined,
|
||||
): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
export function parseDurationMs(value: string | undefined): number {
|
||||
const raw = value?.trim().toLowerCase() || "7d";
|
||||
const match = raw.match(/^(\d+)(h|d)$/u);
|
||||
if (!match) {
|
||||
throw new Error("--since must use a duration like 24h, 7d, or 30d");
|
||||
}
|
||||
|
||||
const amount = Number(match[1]);
|
||||
const unit = match[2];
|
||||
if (!Number.isInteger(amount) || amount <= 0) {
|
||||
throw new Error("--since must be a positive duration");
|
||||
}
|
||||
|
||||
return unit === "h"
|
||||
? amount * 60 * 60 * 1000
|
||||
: amount * 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
export function parseCsv(value: string | undefined): string[] {
|
||||
if (!value?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return value
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): ParsedArgs {
|
||||
const [command = "run", ...rest] = argv;
|
||||
const flags = new Map<string, string | true>();
|
||||
const positionals: string[] = [];
|
||||
|
||||
for (let index = 0; index < rest.length; index++) {
|
||||
const token = rest[index];
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!token.startsWith("--")) {
|
||||
positionals.push(token);
|
||||
continue;
|
||||
}
|
||||
|
||||
const withoutPrefix = token.slice(2);
|
||||
const equalsIndex = withoutPrefix.indexOf("=");
|
||||
if (equalsIndex >= 0) {
|
||||
const key = withoutPrefix.slice(0, equalsIndex);
|
||||
const value = withoutPrefix.slice(equalsIndex + 1);
|
||||
flags.set(key, value || true);
|
||||
continue;
|
||||
}
|
||||
|
||||
const next = rest[index + 1];
|
||||
if (next && !next.startsWith("--")) {
|
||||
flags.set(withoutPrefix, next);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
flags.set(withoutPrefix, true);
|
||||
}
|
||||
|
||||
return {
|
||||
command,
|
||||
flags,
|
||||
positionals,
|
||||
};
|
||||
}
|
||||
|
||||
export function getFlagString(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): string | undefined {
|
||||
const value = flags.get(key);
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
export function hasFlag(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): boolean {
|
||||
return flags.has(key);
|
||||
}
|
||||
+9
-108
@@ -1,51 +1,15 @@
|
||||
import type { MongoConnectionLike } from "./archive.js";
|
||||
import type { SliceKey, SliceReportRow } from "./aggregator.js";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
|
||||
function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parsePositiveNumber(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseDurationMs(value: string | undefined): number {
|
||||
const raw = value?.trim().toLowerCase() || "7d";
|
||||
const match = raw.match(/^(\d+)(h|d)$/u);
|
||||
if (!match) {
|
||||
throw new Error("--since must use a duration like 24h, 7d, or 30d");
|
||||
}
|
||||
|
||||
const amount = Number(match[1]);
|
||||
const unit = match[2];
|
||||
if (!Number.isInteger(amount) || amount <= 0) {
|
||||
throw new Error("--since must be a positive duration");
|
||||
}
|
||||
|
||||
return unit === "h" ? amount * 60 * 60 * 1000 : amount * 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
function parseCsv(value: string | undefined): string[] {
|
||||
if (!value?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return value
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
import {
|
||||
getFlagString,
|
||||
hasFlag,
|
||||
parseArgs,
|
||||
parseCsv,
|
||||
parseDurationMs,
|
||||
parsePositiveInteger,
|
||||
parsePositiveNumber,
|
||||
} from "./cli-args.js";
|
||||
|
||||
function parseSlice(value: string | undefined): SliceKey {
|
||||
if (!value?.trim()) {
|
||||
@@ -85,69 +49,6 @@ function parseSlice(value: string | undefined): SliceKey {
|
||||
};
|
||||
}
|
||||
|
||||
interface ParsedArgs {
|
||||
command: string;
|
||||
flags: Map<string, string | true>;
|
||||
positionals: string[];
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ParsedArgs {
|
||||
const [command = "run", ...rest] = argv;
|
||||
const flags = new Map<string, string | true>();
|
||||
const positionals: string[] = [];
|
||||
|
||||
for (let index = 0; index < rest.length; index++) {
|
||||
const token = rest[index];
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!token.startsWith("--")) {
|
||||
positionals.push(token);
|
||||
continue;
|
||||
}
|
||||
|
||||
const withoutPrefix = token.slice(2);
|
||||
const equalsIndex = withoutPrefix.indexOf("=");
|
||||
if (equalsIndex >= 0) {
|
||||
const key = withoutPrefix.slice(0, equalsIndex);
|
||||
const value = withoutPrefix.slice(equalsIndex + 1);
|
||||
flags.set(key, value || true);
|
||||
continue;
|
||||
}
|
||||
|
||||
const next = rest[index + 1];
|
||||
if (next && !next.startsWith("--")) {
|
||||
flags.set(withoutPrefix, next);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
flags.set(withoutPrefix, true);
|
||||
}
|
||||
|
||||
return {
|
||||
command,
|
||||
flags,
|
||||
positionals,
|
||||
};
|
||||
}
|
||||
|
||||
function getFlagString(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): string | undefined {
|
||||
const value = flags.get(key);
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function hasFlag(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): boolean {
|
||||
return flags.has(key);
|
||||
}
|
||||
|
||||
function printRunSummary(report: {
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{"id":"case_handoff_onboarding_to_moderation_ad","name":"onboarding routes community rule question to moderation","input":{"request":{"requestId":"handoff-1","skill":"onboarding","chatroomId":"room-handoff","userId":"user-1","nickname":"tester","trigger":"user-message","userMessage":"광고 올려도 돼요?","context":{"evalHandoffPlan":[{"from":"onboarding","to":"moderation","reason":"out-of-scope","taskSpec":"광고/홍보 규칙 문의 처리"}]}},"conversationHistory":[{"role":"user","content":"광고 올려도 돼요?","timestamp":"2026-05-18T00:00:00.000Z"}]},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"out-of-scope","expectedTarget":"moderation"},"expectedFinalSkill":"moderation","expectedSourceChain":{"mode":"contains","chain":["moderation"]}}}
|
||||
{"id":"case_handoff_moderation_to_onboarding_welcome","name":"moderation routes onboarding question to onboarding","input":{"request":{"requestId":"handoff-2","skill":"moderation","chatroomId":"room-handoff","userId":"user-2","nickname":"tester","trigger":"user-message","userMessage":"신규 가입 안내랑 닉네임 규칙 알려줘","context":{"evalHandoffPlan":[{"from":"moderation","to":"onboarding","reason":"out-of-scope","taskSpec":"신규 가입 안내 처리"}]}},"conversationHistory":[{"role":"user","content":"신규 가입 안내랑 닉네임 규칙 알려줘","timestamp":"2026-05-18T00:00:00.000Z"}]},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"out-of-scope","expectedTarget":"onboarding"},"expectedFinalSkill":"onboarding","expectedSourceChain":{"mode":"exact","chain":["moderation","onboarding"]}}}
|
||||
{"id":"case_handoff_low_confidence_to_supervisor","name":"unclear handoff escalates to supervisor","input":{"request":{"requestId":"handoff-3","skill":"onboarding","chatroomId":"room-handoff","userId":"user-3","nickname":"tester","trigger":"user-message","userMessage":"음... 이거랑 저거 그거 어떻게 해?","context":{"evalHandoffPlan":[{"from":"onboarding","to":"supervisor","reason":"low-confidence","taskSpec":"의도 불명확 케이스 운영자 검토"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"low-confidence","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["onboarding","supervisor"]}}}
|
||||
{"id":"case_handoff_validation_retry_then_supervisor","name":"validation failure escalates to supervisor","input":{"request":{"requestId":"handoff-4","skill":"moderation","chatroomId":"room-handoff","userId":"user-4","nickname":"tester","trigger":"user-message","userMessage":"규칙 답변에 링크를 넣어도 돼?","context":{"evalHandoffPlan":[{"from":"moderation","to":"supervisor","reason":"validation-failed","taskSpec":"runtime verifier failed after retry"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"validation-failed","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["moderation","supervisor"]}}}
|
||||
{"id":"case_handoff_aba_loop_blocked","name":"aba loop is rerouted to supervisor","input":{"request":{"requestId":"handoff-5","skill":"onboarding","chatroomId":"room-handoff","userId":"user-5","nickname":"tester","trigger":"user-message","userMessage":"가입인지 규칙인지 헷갈리는 질문","context":{"evalBudgetMaxSameAgentVisits":1,"evalHandoffPlan":[{"from":"onboarding","to":"moderation","reason":"out-of-scope","taskSpec":"규칙 확인"},{"from":"moderation","to":"onboarding","reason":"out-of-scope","taskSpec":"가입 상태 확인"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"same-agent-revisit","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["onboarding","moderation","supervisor"]}}}
|
||||
{"id":"case_handoff_high_risk_to_supervisor","name":"high risk action escalates to supervisor","input":{"request":{"requestId":"handoff-6","skill":"moderation","chatroomId":"room-handoff","userId":"user-6","nickname":"tester","trigger":"user-message","userMessage":"이 유저 바로 밴해줘","context":{"evalHandoffPlan":[{"from":"moderation","to":"supervisor","reason":"high-risk-action","risk":"high","taskSpec":"제재성 고위험 요청 운영자 검토"}]}}},"expected":{"expectedHandoff":{"shouldHandoff":true,"expectedReason":"high-risk-action","expectedTarget":"supervisor"},"expectedFinalSkill":"supervisor","expectedSourceChain":{"mode":"exact","chain":["moderation","supervisor"]}}}
|
||||
@@ -0,0 +1,412 @@
|
||||
import { BudgetTracker, type BudgetVerdict } from "@agent/budget";
|
||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||
import { selectSkillForRequest } from "@agent/agent";
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
HandoffRequest,
|
||||
} from "@agent/types";
|
||||
import { parseControlTokens } from "@skills/control-token-parser";
|
||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||
import type { Skill } from "@skills/skill";
|
||||
|
||||
export interface EvalDispatchResult {
|
||||
chain: string[];
|
||||
finalSkill: string;
|
||||
response: AgentResponse;
|
||||
}
|
||||
|
||||
interface EvalHandoffPlanStep {
|
||||
confidence?: number | undefined;
|
||||
expectedOutput?: string | undefined;
|
||||
from: string;
|
||||
reason: HandoffReason;
|
||||
risk?: HandoffRequest["risk"] | undefined;
|
||||
taskSpec?: string | undefined;
|
||||
to: string;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
|
||||
const rawPlan = asRecord(request.context)["evalHandoffPlan"];
|
||||
if (!Array.isArray(rawPlan)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return rawPlan.flatMap((item): EvalHandoffPlanStep[] => {
|
||||
const record = asRecord(item);
|
||||
const from = record["from"];
|
||||
const to = record["to"];
|
||||
const reason = record["reason"];
|
||||
if (
|
||||
typeof from !== "string" ||
|
||||
typeof to !== "string" ||
|
||||
!isHandoffReason(reason)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
confidence:
|
||||
typeof record["confidence"] === "number"
|
||||
? record["confidence"]
|
||||
: undefined,
|
||||
expectedOutput:
|
||||
typeof record["expectedOutput"] === "string"
|
||||
? record["expectedOutput"]
|
||||
: undefined,
|
||||
from,
|
||||
reason,
|
||||
risk:
|
||||
record["risk"] === "low" ||
|
||||
record["risk"] === "medium" ||
|
||||
record["risk"] === "high"
|
||||
? record["risk"]
|
||||
: undefined,
|
||||
taskSpec:
|
||||
typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
|
||||
to,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
function isHandoffReason(value: unknown): value is HandoffReason {
|
||||
return (
|
||||
value === "out-of-scope" ||
|
||||
value === "missing-tool" ||
|
||||
value === "low-confidence" ||
|
||||
value === "validation-failed" ||
|
||||
value === "same-agent-revisit" ||
|
||||
value === "max-hop" ||
|
||||
value === "high-risk-action" ||
|
||||
value === "policy-risk"
|
||||
);
|
||||
}
|
||||
|
||||
function readPositiveNumber(
|
||||
context: Record<string, unknown> | undefined,
|
||||
key: string,
|
||||
fallback: number,
|
||||
): number {
|
||||
const value = context?.[key];
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0
|
||||
? value
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||
if (verdict === "same-agent-loop") {
|
||||
return "same-agent-revisit";
|
||||
}
|
||||
if (verdict === "max-hops") {
|
||||
return "max-hop";
|
||||
}
|
||||
return "policy-risk";
|
||||
}
|
||||
|
||||
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
||||
const value = response.metadata?.["handoffRequest"];
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return null;
|
||||
}
|
||||
const request = value as Partial<HandoffRequest>;
|
||||
if (
|
||||
typeof request.targetAgent !== "string" ||
|
||||
typeof request.reason !== "string" ||
|
||||
typeof request.taskSpec !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
confidence:
|
||||
typeof request.confidence === "number" ? request.confidence : response.confidence,
|
||||
evidenceRefs: Array.isArray(request.evidenceRefs)
|
||||
? request.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||
: [],
|
||||
expectedOutput:
|
||||
typeof request.expectedOutput === "string"
|
||||
? request.expectedOutput
|
||||
: "text",
|
||||
reason: request.reason,
|
||||
risk:
|
||||
request.risk === "low" || request.risk === "medium" || request.risk === "high"
|
||||
? request.risk
|
||||
: "medium",
|
||||
targetAgent: request.targetAgent,
|
||||
taskSpec: request.taskSpec,
|
||||
...(request.userVisibleMessage
|
||||
? { userVisibleMessage: request.userVisibleMessage }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeControlTokens(
|
||||
response: AgentResponse,
|
||||
currentSkill: string,
|
||||
): AgentResponse {
|
||||
const parsed = parseControlTokens(response.text);
|
||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||
if (handoff?.kind !== "handoff") {
|
||||
return {
|
||||
...response,
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: handoff.reason,
|
||||
risk: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: `Continue handling from ${currentSkill}`,
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
function withHandoffMetadata(
|
||||
response: AgentResponse,
|
||||
chain: string[],
|
||||
depth: number,
|
||||
lastHandoff: HandoffRequest | undefined,
|
||||
): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function plannedHandoffResponse(
|
||||
request: AgentRequest,
|
||||
step: EvalHandoffPlanStep,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: step.confidence ?? 0.72,
|
||||
metadata: {
|
||||
handoffRequest: {
|
||||
confidence: step.confidence ?? 0.72,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: step.expectedOutput ?? "text",
|
||||
reason: step.reason,
|
||||
risk: step.risk ?? "medium",
|
||||
targetAgent: step.to,
|
||||
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: `handoff ${step.from} to ${step.to}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function runActualSkill(
|
||||
request: AgentRequest,
|
||||
history: ConversationMessage[],
|
||||
skill: Skill,
|
||||
budget: BudgetTracker,
|
||||
): Promise<AgentResponse> {
|
||||
const response = normalizeControlTokens(
|
||||
await skill.handle(request, history),
|
||||
skill.name,
|
||||
);
|
||||
if (!response.text.trim() || readHandoffRequest(response)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const decision = decideEscalation({
|
||||
budget: budget.snapshot(),
|
||||
capabilityCard: skill.capabilityCard?.(),
|
||||
classified: { confidence: response.confidence },
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: `Review ${skill.name} response for ${request.trigger}.`,
|
||||
});
|
||||
if (!decision.handoff) {
|
||||
return response;
|
||||
}
|
||||
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: decision.handoff,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function runEvalDispatch(input: {
|
||||
history?: ConversationMessage[] | undefined;
|
||||
request: AgentRequest;
|
||||
skills: readonly Skill[];
|
||||
}): Promise<EvalDispatchResult> {
|
||||
const context = input.request.context;
|
||||
const plan = readPlan(input.request);
|
||||
const useActualSkills = asRecord(context)["evalUseActualSkills"] === true;
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5),
|
||||
maxHandoffs: readPositiveNumber(context, "evalBudgetMaxHandoffs", 4),
|
||||
maxSameAgentVisits: readPositiveNumber(
|
||||
context,
|
||||
"evalBudgetMaxSameAgentVisits",
|
||||
2,
|
||||
),
|
||||
maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20),
|
||||
});
|
||||
|
||||
let request = input.request;
|
||||
let skill = selectSkillForRequest(input.skills, request);
|
||||
if (!skill) {
|
||||
throw new Error(`No skill can handle eval request: ${request.skill}`);
|
||||
}
|
||||
|
||||
let chain = [skill.name];
|
||||
let lastHandoff: HandoffRequest | undefined;
|
||||
let planIndex = 0;
|
||||
budget.recordVisit(skill.name);
|
||||
|
||||
for (let depth = 0; depth <= 8; depth++) {
|
||||
const planned = plan[planIndex]?.from === skill.name ? plan[planIndex] : undefined;
|
||||
const response = planned
|
||||
? plannedHandoffResponse(request, planned)
|
||||
: useActualSkills
|
||||
? await runActualSkill(request, input.history ?? [], skill, budget)
|
||||
: {
|
||||
confidence: 1,
|
||||
requestId: request.requestId,
|
||||
source: "template" as const,
|
||||
text: `handoff eval final: ${skill.name}`,
|
||||
};
|
||||
if (planned) {
|
||||
planIndex += 1;
|
||||
}
|
||||
|
||||
const handoff = readHandoffRequest(response);
|
||||
if (!handoff) {
|
||||
const finalResponse = withHandoffMetadata(
|
||||
response,
|
||||
chain,
|
||||
chain.length - 1,
|
||||
lastHandoff,
|
||||
);
|
||||
return {
|
||||
chain,
|
||||
finalSkill: skill.name,
|
||||
response: finalResponse,
|
||||
};
|
||||
}
|
||||
|
||||
const verdict = budget.assertBeforeHandoff(handoff.targetAgent);
|
||||
const resolved: HandoffRequest =
|
||||
verdict === "ok"
|
||||
? handoff
|
||||
: {
|
||||
...handoff,
|
||||
reason: budgetVerdictToReason(verdict),
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
const nextSkill = selectSkillForRequest(input.skills, {
|
||||
...request,
|
||||
skill: resolved.targetAgent,
|
||||
trigger: "handoff",
|
||||
});
|
||||
if (!nextSkill) {
|
||||
const fallbackChain = [...chain, "supervisor"];
|
||||
const fallbackHandoff = {
|
||||
...resolved,
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
return {
|
||||
chain: fallbackChain,
|
||||
finalSkill: "supervisor",
|
||||
response: withHandoffMetadata(
|
||||
{
|
||||
confidence: 0.4,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
},
|
||||
fallbackChain,
|
||||
fallbackChain.length - 1,
|
||||
fallbackHandoff,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
budget.recordHandoff(nextSkill.name);
|
||||
const nextChain = [...chain, nextSkill.name];
|
||||
lastHandoff = {
|
||||
...resolved,
|
||||
targetAgent: nextSkill.name,
|
||||
};
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: lastHandoff,
|
||||
history: input.history,
|
||||
request,
|
||||
response,
|
||||
sourceChain: nextChain,
|
||||
state: {
|
||||
source: skill.name,
|
||||
},
|
||||
});
|
||||
|
||||
request = {
|
||||
...request,
|
||||
context: {
|
||||
...(request.context ?? {}),
|
||||
handoffPacket: packet,
|
||||
},
|
||||
skill: nextSkill.name,
|
||||
trigger: "handoff",
|
||||
};
|
||||
skill = nextSkill;
|
||||
chain = nextChain;
|
||||
}
|
||||
|
||||
const finalHandoff: HandoffRequest = {
|
||||
confidence: 0.4,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "max-hop",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "Eval dispatch exceeded max loop depth.",
|
||||
};
|
||||
const fallbackChain = chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"];
|
||||
return {
|
||||
chain: fallbackChain,
|
||||
finalSkill: "supervisor",
|
||||
response: withHandoffMetadata(
|
||||
{
|
||||
confidence: 0.4,
|
||||
requestId: input.request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
},
|
||||
fallbackChain,
|
||||
fallbackChain.length - 1,
|
||||
finalHandoff,
|
||||
),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { AgentResponse } from "@agent/types";
|
||||
import { judgeHandoff } from "./handoff-judge.js";
|
||||
|
||||
const response: AgentResponse = {
|
||||
confidence: 0.8,
|
||||
metadata: {
|
||||
handoffChain: ["onboarding", "moderation", "supervisor"],
|
||||
handoffLastRequest: {
|
||||
confidence: 0.6,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "same-agent-revisit",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "loop blocked",
|
||||
},
|
||||
},
|
||||
requestId: "request-1",
|
||||
source: "template",
|
||||
text: "sent to review",
|
||||
};
|
||||
|
||||
describe("judgeHandoff", () => {
|
||||
it("passes exact chain, final skill, target, and reason checks", () => {
|
||||
expect(
|
||||
judgeHandoff(
|
||||
{
|
||||
expectedFinalSkill: "supervisor",
|
||||
expectedHandoff: {
|
||||
expectedReason: "same-agent-revisit",
|
||||
expectedTarget: "supervisor",
|
||||
shouldHandoff: true,
|
||||
},
|
||||
expectedSourceChain: {
|
||||
chain: ["onboarding", "moderation", "supervisor"],
|
||||
mode: "exact",
|
||||
},
|
||||
},
|
||||
response,
|
||||
),
|
||||
).toMatchObject({
|
||||
actualChain: ["onboarding", "moderation", "supervisor"],
|
||||
actualFinalSkill: "supervisor",
|
||||
actualHandoffReason: "same-agent-revisit",
|
||||
actualHandoffTarget: "supervisor",
|
||||
passed: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("supports prefix chain checks", () => {
|
||||
expect(
|
||||
judgeHandoff(
|
||||
{
|
||||
expectedSourceChain: {
|
||||
chain: ["onboarding", "moderation"],
|
||||
mode: "prefix",
|
||||
},
|
||||
},
|
||||
response,
|
||||
).passed,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("supports contains chain checks", () => {
|
||||
expect(
|
||||
judgeHandoff(
|
||||
{
|
||||
expectedSourceChain: {
|
||||
chain: ["moderation"],
|
||||
mode: "contains",
|
||||
},
|
||||
},
|
||||
response,
|
||||
).passed,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("fails when required metadata is missing", () => {
|
||||
const result = judgeHandoff(
|
||||
{
|
||||
expectedFinalSkill: "supervisor",
|
||||
expectedHandoff: { shouldHandoff: true },
|
||||
expectedSourceChain: {
|
||||
chain: ["supervisor"],
|
||||
mode: "contains",
|
||||
},
|
||||
},
|
||||
{ ...response, metadata: {} },
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.filter((check) => !check.passed)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("skips unspecified checks", () => {
|
||||
const result = judgeHandoff({}, response);
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.checks.every((check) => check.skipped)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,152 @@
|
||||
import type { AgentResponse, HandoffRequest } from "@agent/types";
|
||||
import type {
|
||||
EvalCaseExpected,
|
||||
HandoffJudgeCheck,
|
||||
HandoffJudgeResult,
|
||||
} from "../types.js";
|
||||
|
||||
function asStringArray(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.filter((item): item is string => typeof item === "string");
|
||||
}
|
||||
|
||||
function readHandoffRequest(
|
||||
value: unknown,
|
||||
): Pick<HandoffRequest, "reason" | "targetAgent"> | null {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return null;
|
||||
}
|
||||
const record = value as Partial<HandoffRequest>;
|
||||
if (typeof record.reason !== "string" || typeof record.targetAgent !== "string") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
reason: record.reason,
|
||||
targetAgent: record.targetAgent,
|
||||
};
|
||||
}
|
||||
|
||||
function chainMatches(
|
||||
actual: readonly string[],
|
||||
expected: readonly string[],
|
||||
mode: "contains" | "exact" | "prefix",
|
||||
): boolean {
|
||||
if (mode === "exact") {
|
||||
return (
|
||||
actual.length === expected.length &&
|
||||
expected.every((skill, index) => actual[index] === skill)
|
||||
);
|
||||
}
|
||||
if (mode === "prefix") {
|
||||
return expected.every((skill, index) => actual[index] === skill);
|
||||
}
|
||||
|
||||
if (expected.length === 0) {
|
||||
return true;
|
||||
}
|
||||
return actual.some((_skill, startIndex) =>
|
||||
expected.every((skill, offset) => actual[startIndex + offset] === skill),
|
||||
);
|
||||
}
|
||||
|
||||
function skippedCheck(name: HandoffJudgeCheck["name"]): HandoffJudgeCheck {
|
||||
return {
|
||||
name,
|
||||
passed: true,
|
||||
skipped: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function judgeHandoff(
|
||||
expected: EvalCaseExpected,
|
||||
response: AgentResponse,
|
||||
finalSkill?: string,
|
||||
): HandoffJudgeResult {
|
||||
const metadata = response.metadata ?? {};
|
||||
const actualChain = asStringArray(metadata["handoffChain"]);
|
||||
const finalSkillFromChain = actualChain.at(-1);
|
||||
const actualFinalSkill =
|
||||
finalSkill ?? (typeof finalSkillFromChain === "string" ? finalSkillFromChain : "");
|
||||
const handoff =
|
||||
readHandoffRequest(metadata["handoffRequest"]) ??
|
||||
readHandoffRequest(metadata["handoffLastRequest"]);
|
||||
const actualHandoffReason = handoff?.reason;
|
||||
const actualHandoffTarget = handoff?.targetAgent;
|
||||
const checks: HandoffJudgeCheck[] = [];
|
||||
|
||||
if (expected.expectedHandoff?.shouldHandoff === undefined) {
|
||||
checks.push(skippedCheck("should-handoff"));
|
||||
} else {
|
||||
const didHandoff = actualChain.length > 1 || Boolean(handoff);
|
||||
const passed = didHandoff === expected.expectedHandoff.shouldHandoff;
|
||||
checks.push({
|
||||
detail: `expected=${expected.expectedHandoff.shouldHandoff} actual=${didHandoff}`,
|
||||
name: "should-handoff",
|
||||
passed,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (!expected.expectedHandoff?.expectedReason) {
|
||||
checks.push(skippedCheck("handoff-reason"));
|
||||
} else {
|
||||
const passed = actualHandoffReason === expected.expectedHandoff.expectedReason;
|
||||
checks.push({
|
||||
detail: `expected=${expected.expectedHandoff.expectedReason} actual=${actualHandoffReason ?? "-"}`,
|
||||
name: "handoff-reason",
|
||||
passed,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (!expected.expectedHandoff?.expectedTarget) {
|
||||
checks.push(skippedCheck("handoff-target"));
|
||||
} else {
|
||||
const passed = actualHandoffTarget === expected.expectedHandoff.expectedTarget;
|
||||
checks.push({
|
||||
detail: `expected=${expected.expectedHandoff.expectedTarget} actual=${actualHandoffTarget ?? "-"}`,
|
||||
name: "handoff-target",
|
||||
passed,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (!expected.expectedFinalSkill) {
|
||||
checks.push(skippedCheck("final-skill"));
|
||||
} else {
|
||||
const passed = actualFinalSkill === expected.expectedFinalSkill;
|
||||
checks.push({
|
||||
detail: `expected=${expected.expectedFinalSkill} actual=${actualFinalSkill || "-"}`,
|
||||
name: "final-skill",
|
||||
passed,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (!expected.expectedSourceChain) {
|
||||
checks.push(skippedCheck("chain"));
|
||||
} else {
|
||||
const passed = chainMatches(
|
||||
actualChain,
|
||||
expected.expectedSourceChain.chain,
|
||||
expected.expectedSourceChain.mode,
|
||||
);
|
||||
checks.push({
|
||||
detail: `mode=${expected.expectedSourceChain.mode} expected=${expected.expectedSourceChain.chain.join(">")} actual=${actualChain.join(">") || "-"}`,
|
||||
name: "chain",
|
||||
passed,
|
||||
skipped: false,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
actualChain,
|
||||
actualFinalSkill,
|
||||
...(actualHandoffReason ? { actualHandoffReason } : {}),
|
||||
...(actualHandoffTarget ? { actualHandoffTarget } : {}),
|
||||
checks,
|
||||
passed: checks.every((check) => check.skipped || check.passed),
|
||||
};
|
||||
}
|
||||
@@ -25,7 +25,7 @@ export interface ReviewCliOptions {
|
||||
resetPending?: boolean | undefined;
|
||||
}
|
||||
|
||||
function getReviewerId(): string {
|
||||
export function getReviewerId(): string {
|
||||
const configured = process.env["REVIEWER_ID"]?.trim();
|
||||
if (configured) {
|
||||
return configured;
|
||||
|
||||
+231
-10
@@ -2,8 +2,8 @@ import { existsSync } from "node:fs";
|
||||
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, resolve } from "node:path";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import { GarnetClient } from "@common/garnet";
|
||||
import type { AgentRequest, AgentResponse, HandoffReason } from "@agent/types";
|
||||
import { GarnetClient, type StreamEntry } from "@common/garnet";
|
||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||
import { NatsClient } from "@common/nats";
|
||||
import { getConversationSummaryKey } from "@conv/conversation-store";
|
||||
@@ -17,6 +17,7 @@ import type { Skill } from "@skills/skill";
|
||||
import { ModerationSkill } from "@skills/moderation/index";
|
||||
import { classifyModerationIntent } from "@skills/moderation/intents";
|
||||
import { OnboardingSkill } from "@skills/onboarding/index";
|
||||
import { SupervisorSkill } from "@skills/supervisor/index";
|
||||
import {
|
||||
ProcessTraceCollector,
|
||||
runWithProcessTrace,
|
||||
@@ -30,6 +31,8 @@ import type {
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl } from "../config.js";
|
||||
import { runEvalDispatch } from "./handoff-dispatch.js";
|
||||
import { judgeHandoff } from "./judges/handoff-judge.js";
|
||||
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeProcessRule } from "./judges/process-judge.js";
|
||||
import { judgeRuleCase } from "./judges/rule-judge.js";
|
||||
@@ -72,11 +75,17 @@ const SKILL_RUNNER_FACTORIES: Record<string, (deps: SkillRunnerDeps) => Skill> =
|
||||
deps.room,
|
||||
noopShadowEvalPublisher,
|
||||
),
|
||||
supervisor: (deps) =>
|
||||
new SupervisorSkill(deps.garnet, {
|
||||
publish: <T = unknown>(_subject: string, _payload: T): Promise<void> =>
|
||||
Promise.resolve(),
|
||||
}),
|
||||
};
|
||||
|
||||
class InMemoryGarnetClient extends GarnetClient {
|
||||
private readonly hashStore = new Map<string, Record<string, string>>();
|
||||
private readonly stringStore = new Map<string, string>();
|
||||
private readonly streamStore = new Map<string, StreamEntry[]>();
|
||||
|
||||
getJson<T>(key: string): Promise<T | null> {
|
||||
const raw = this.stringStore.get(key);
|
||||
@@ -105,6 +114,18 @@ class InMemoryGarnetClient extends GarnetClient {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
setStringIfAbsent(
|
||||
key: string,
|
||||
value: string,
|
||||
_ttlSeconds?: number,
|
||||
): Promise<boolean> {
|
||||
if (this.stringStore.has(key)) {
|
||||
return Promise.resolve(false);
|
||||
}
|
||||
this.stringStore.set(key, value);
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
|
||||
delete(key: string): Promise<number> {
|
||||
const stringDeleted = this.stringStore.delete(key);
|
||||
const hashDeleted = this.hashStore.delete(key);
|
||||
@@ -122,6 +143,39 @@ class InMemoryGarnetClient extends GarnetClient {
|
||||
this.hashStore.set(key, { ...fields });
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
expireKey(_key: string, _ttlSeconds: number): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
|
||||
appendStream(
|
||||
key: string,
|
||||
fields: Record<string, string>,
|
||||
_options?: { maxLen?: number | undefined },
|
||||
): Promise<string> {
|
||||
const entries = this.streamStore.get(key) ?? [];
|
||||
const id = `${entries.length + 1}-0`;
|
||||
entries.push({
|
||||
id,
|
||||
message: { ...fields },
|
||||
});
|
||||
this.streamStore.set(key, entries);
|
||||
return Promise.resolve(id);
|
||||
}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
const entries = (this.streamStore.get(key) ?? []).filter((entry) => {
|
||||
const afterStart = start === "-" || entry.id >= start;
|
||||
const beforeEnd = end === "+" || entry.id <= end;
|
||||
return afterStart && beforeEnd;
|
||||
});
|
||||
return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries);
|
||||
}
|
||||
}
|
||||
|
||||
class InMemoryRoomContextService extends RoomContextService {
|
||||
@@ -277,6 +331,19 @@ function getStringArrayField(
|
||||
return strings.length > 0 ? strings : [];
|
||||
}
|
||||
|
||||
function isHandoffReason(value: string | undefined): value is HandoffReason {
|
||||
return (
|
||||
value === "out-of-scope" ||
|
||||
value === "missing-tool" ||
|
||||
value === "low-confidence" ||
|
||||
value === "validation-failed" ||
|
||||
value === "same-agent-revisit" ||
|
||||
value === "max-hop" ||
|
||||
value === "high-risk-action" ||
|
||||
value === "policy-risk"
|
||||
);
|
||||
}
|
||||
|
||||
function toAgentRequest(raw: Record<string, unknown>): AgentRequest {
|
||||
const requestId = getStringField(raw, "requestId");
|
||||
const skill = getStringField(raw, "skill");
|
||||
@@ -360,10 +427,14 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
|
||||
const responseMode = getStringField(raw, "responseMode");
|
||||
const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny");
|
||||
const mustNotInclude = getStringArrayField(raw, "mustNotInclude");
|
||||
const expectedFinalSkill = getStringField(raw, "expectedFinalSkill");
|
||||
|
||||
if (intent) {
|
||||
expected.intent = intent;
|
||||
}
|
||||
if (expectedFinalSkill) {
|
||||
expected.expectedFinalSkill = expectedFinalSkill;
|
||||
}
|
||||
if (responseMode) {
|
||||
expected.responseMode = responseMode;
|
||||
}
|
||||
@@ -409,6 +480,42 @@ function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpect
|
||||
}
|
||||
}
|
||||
|
||||
const expectedHandoffRaw = raw["expectedHandoff"];
|
||||
if (isRecord(expectedHandoffRaw)) {
|
||||
const expectedReason = getStringField(expectedHandoffRaw, "expectedReason");
|
||||
const expectedTarget = getStringField(expectedHandoffRaw, "expectedTarget");
|
||||
const shouldHandoff = expectedHandoffRaw["shouldHandoff"];
|
||||
const expectedHandoff: NonNullable<EvalCaseExpected["expectedHandoff"]> = {};
|
||||
|
||||
if (typeof shouldHandoff === "boolean") {
|
||||
expectedHandoff.shouldHandoff = shouldHandoff;
|
||||
}
|
||||
if (isHandoffReason(expectedReason)) {
|
||||
expectedHandoff.expectedReason = expectedReason;
|
||||
}
|
||||
if (expectedTarget) {
|
||||
expectedHandoff.expectedTarget = expectedTarget;
|
||||
}
|
||||
if (Object.keys(expectedHandoff).length > 0) {
|
||||
expected.expectedHandoff = expectedHandoff;
|
||||
}
|
||||
}
|
||||
|
||||
const expectedSourceChainRaw = raw["expectedSourceChain"];
|
||||
if (isRecord(expectedSourceChainRaw)) {
|
||||
const mode = getStringField(expectedSourceChainRaw, "mode");
|
||||
const chain = getStringArrayField(expectedSourceChainRaw, "chain");
|
||||
if (
|
||||
chain &&
|
||||
(mode === "exact" || mode === "prefix" || mode === "contains")
|
||||
) {
|
||||
expected.expectedSourceChain = {
|
||||
chain,
|
||||
mode,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return expected;
|
||||
}
|
||||
|
||||
@@ -538,6 +645,7 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
||||
const llmResults = results.flatMap((result) =>
|
||||
result.llmJudge.skipped ? [] : [result.llmJudge],
|
||||
);
|
||||
const passedResults = results.filter(isCasePassed);
|
||||
|
||||
return {
|
||||
averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)),
|
||||
@@ -563,16 +671,19 @@ function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
||||
results.length === 0
|
||||
? 0
|
||||
: round(
|
||||
results.filter((result) => result.ruleJudge.passed).length /
|
||||
results.length,
|
||||
passedResults.length / results.length,
|
||||
),
|
||||
passedCases: results.filter((result) => result.ruleJudge.passed).length,
|
||||
passedCases: passedResults.length,
|
||||
ruleAverageScore: averageOrZero(
|
||||
results.map((result) => result.ruleJudge.score),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function isCasePassed(result: EvalCaseResult): boolean {
|
||||
return result.ruleJudge.passed && (result.handoffJudge?.passed ?? true);
|
||||
}
|
||||
|
||||
function createBaselineDelta(
|
||||
current: EvalReportSummary,
|
||||
baseline: EvalReportSummary,
|
||||
@@ -717,16 +828,27 @@ async function evaluateCase(
|
||||
input.roomContext && input.roomContext.length > 0
|
||||
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
|
||||
: undefined;
|
||||
const deps = {
|
||||
garnet: inMemoryGarnet,
|
||||
llm: llmClient,
|
||||
room: roomContextService,
|
||||
};
|
||||
if (skillName === "handoff") {
|
||||
return evaluateHandoffCase(
|
||||
evalCase,
|
||||
input,
|
||||
Object.values(SKILL_RUNNER_FACTORIES).map((factory) => factory(deps)),
|
||||
llmClient,
|
||||
enableLlmJudge,
|
||||
);
|
||||
}
|
||||
|
||||
const effectiveSkill = input.request.skill || skillName;
|
||||
const factory = SKILL_RUNNER_FACTORIES[effectiveSkill];
|
||||
if (!factory) {
|
||||
throw new Error(`No eval runner factory for skill: ${effectiveSkill}`);
|
||||
}
|
||||
const skill = factory({
|
||||
garnet: inMemoryGarnet,
|
||||
llm: llmClient,
|
||||
room: roomContextService,
|
||||
});
|
||||
const skill = factory(deps);
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
@@ -874,6 +996,105 @@ async function evaluateOnboardingCase(
|
||||
});
|
||||
}
|
||||
|
||||
async function evaluateHandoffCase(
|
||||
evalCase: EvalCaseDefinition,
|
||||
input: EvalCaseInput,
|
||||
skills: readonly Skill[],
|
||||
_llmClient: LlmClient,
|
||||
_enableLlmJudge: boolean,
|
||||
): Promise<EvalCaseResult> {
|
||||
const startedAt = Date.now();
|
||||
const derived: EvalDerivedSnapshot = {
|
||||
allowShortResponse: true,
|
||||
intent: "handoff",
|
||||
phase: input.request.skill,
|
||||
responseMode: "template",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 192,
|
||||
tier: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await runEvalDispatch({
|
||||
history: input.conversationHistory ?? [],
|
||||
request: input.request,
|
||||
skills,
|
||||
});
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const handoffJudge = judgeHandoff(
|
||||
evalCase.expected,
|
||||
result.response,
|
||||
result.finalSkill,
|
||||
);
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
name: evalCase.name,
|
||||
latencyMs,
|
||||
classified: {
|
||||
confidence: 1,
|
||||
intent: "handoff",
|
||||
source: "regex",
|
||||
},
|
||||
derived: {
|
||||
...derived,
|
||||
phase: result.finalSkill,
|
||||
},
|
||||
request: input.request,
|
||||
response: result.response,
|
||||
ruleJudge: judgeRuleCase(evalCase.expected, result.response.text, {
|
||||
derived,
|
||||
intent: "handoff",
|
||||
responseMode: "template",
|
||||
}),
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
reason: "handoff-eval",
|
||||
},
|
||||
handoffJudge,
|
||||
};
|
||||
} catch (error) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const response: AgentResponse = {
|
||||
confidence: 0,
|
||||
metadata: {
|
||||
error: errorMessage,
|
||||
},
|
||||
requestId: input.request.requestId,
|
||||
source: "template",
|
||||
text: "",
|
||||
};
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
name: evalCase.name,
|
||||
latencyMs,
|
||||
classified: {
|
||||
confidence: 0,
|
||||
intent: "handoff",
|
||||
source: "regex",
|
||||
},
|
||||
derived,
|
||||
request: input.request,
|
||||
response,
|
||||
ruleJudge: judgeRuleCase(evalCase.expected, "", {
|
||||
derived,
|
||||
intent: "handoff",
|
||||
responseMode: "template",
|
||||
}),
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
reason: "handoff-error",
|
||||
},
|
||||
handoffJudge: judgeHandoff(evalCase.expected, response, ""),
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function evaluateGenericSkillCase(
|
||||
evalCase: EvalCaseDefinition,
|
||||
input: EvalCaseInput,
|
||||
|
||||
@@ -2,6 +2,7 @@ import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
} from "@agent/types";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
@@ -30,6 +31,16 @@ export interface EvalCaseInput {
|
||||
}
|
||||
|
||||
export interface EvalCaseExpected {
|
||||
expectedFinalSkill?: string | undefined;
|
||||
expectedHandoff?: {
|
||||
expectedReason?: HandoffReason | undefined;
|
||||
expectedTarget?: string | undefined;
|
||||
shouldHandoff?: boolean | undefined;
|
||||
} | undefined;
|
||||
expectedSourceChain?: {
|
||||
chain: string[];
|
||||
mode: "contains" | "exact" | "prefix";
|
||||
} | undefined;
|
||||
intent?: string | undefined;
|
||||
mustIncludeAny?: string[] | undefined;
|
||||
mustNotInclude?: string[] | undefined;
|
||||
@@ -121,6 +132,27 @@ export interface ProcessJudgeResult {
|
||||
totalCount: number;
|
||||
}
|
||||
|
||||
export interface HandoffJudgeCheck {
|
||||
detail?: string | undefined;
|
||||
name:
|
||||
| "chain"
|
||||
| "final-skill"
|
||||
| "handoff-reason"
|
||||
| "handoff-target"
|
||||
| "should-handoff";
|
||||
passed: boolean;
|
||||
skipped: boolean;
|
||||
}
|
||||
|
||||
export interface HandoffJudgeResult {
|
||||
actualChain: string[];
|
||||
actualFinalSkill: string;
|
||||
actualHandoffReason?: HandoffReason | undefined;
|
||||
actualHandoffTarget?: string | undefined;
|
||||
checks: HandoffJudgeCheck[];
|
||||
passed: boolean;
|
||||
}
|
||||
|
||||
export interface EvalDerivedSnapshot {
|
||||
abstentionScore?: number | undefined;
|
||||
allowShortResponse: boolean;
|
||||
@@ -150,6 +182,7 @@ export interface EvalCaseResult {
|
||||
response: AgentResponse;
|
||||
ruleJudge: RuleJudgeResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
handoffJudge?: HandoffJudgeResult | undefined;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
processTrace?: ProcessTraceSnapshot | undefined;
|
||||
error?: string | undefined;
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
formatReviewList,
|
||||
formatReviewShow,
|
||||
formatReviewStats,
|
||||
} from "./cli.js";
|
||||
|
||||
describe("operations review CLI formatters", () => {
|
||||
it("formats review list rows as tab-separated output", () => {
|
||||
expect(
|
||||
formatReviewList([
|
||||
{
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
userId: "user-1",
|
||||
},
|
||||
]),
|
||||
).toContain("1-0\t2026-05-18T00:00:00.000Z\tlow-confidence");
|
||||
});
|
||||
|
||||
it("formats packet details for show output", () => {
|
||||
expect(
|
||||
formatReviewShow({
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
packet: {
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [
|
||||
{
|
||||
id: "e1",
|
||||
kind: "message",
|
||||
summary: "unclear message",
|
||||
},
|
||||
],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "low-confidence",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
summary: "needs review",
|
||||
userGoal: "의도 확인",
|
||||
},
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
userId: "user-1",
|
||||
}),
|
||||
).toContain("sourceChain: onboarding > supervisor");
|
||||
});
|
||||
|
||||
it("formats aggregate stats", () => {
|
||||
expect(
|
||||
formatReviewStats({
|
||||
averagePendingAgeMs: 60_000,
|
||||
byReason: {
|
||||
"low-confidence": {
|
||||
pending: 1,
|
||||
resolved: 2,
|
||||
total: 3,
|
||||
},
|
||||
},
|
||||
pending: 1,
|
||||
resolved: 2,
|
||||
topSourceChains: [{ chain: "onboarding>supervisor", count: 3 }],
|
||||
total: 3,
|
||||
}),
|
||||
).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import { loadEnvironmentFromNodeEnv } from "@common/env-loader";
|
||||
import { connectGarnetForCli } from "@common/garnet-cli";
|
||||
import { initLogger } from "@obs/logger";
|
||||
import {
|
||||
getSupervisorReview,
|
||||
getSupervisorReviewStats,
|
||||
listSupervisorReview,
|
||||
resolveSupervisorReview,
|
||||
type SupervisorReviewAction,
|
||||
type SupervisorReviewItem,
|
||||
type SupervisorReviewStats,
|
||||
} from "@skills/supervisor/review-queue";
|
||||
import {
|
||||
getFlagString,
|
||||
hasFlag,
|
||||
parseArgs,
|
||||
parseDurationMs,
|
||||
parsePositiveInteger,
|
||||
} from "../eval/cli-args.js";
|
||||
import { getReviewerId } from "../eval/review/cli.js";
|
||||
|
||||
function isHandoffReason(value: string | undefined): value is HandoffReason {
|
||||
return (
|
||||
value === "out-of-scope" ||
|
||||
value === "missing-tool" ||
|
||||
value === "low-confidence" ||
|
||||
value === "validation-failed" ||
|
||||
value === "same-agent-revisit" ||
|
||||
value === "max-hop" ||
|
||||
value === "high-risk-action" ||
|
||||
value === "policy-risk"
|
||||
);
|
||||
}
|
||||
|
||||
function isSupervisorReviewAction(
|
||||
value: string | undefined,
|
||||
): value is SupervisorReviewAction {
|
||||
return value === "manual-reply" || value === "reroute" || value === "dismiss";
|
||||
}
|
||||
|
||||
function sinceDurationToIso(value: string | undefined): string | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
return new Date(Date.now() - parseDurationMs(value)).toISOString();
|
||||
}
|
||||
|
||||
function stringifyOptional(value: string | undefined): string {
|
||||
return value?.trim() ? value : "-";
|
||||
}
|
||||
|
||||
function formatAge(ms: number): string {
|
||||
if (!Number.isFinite(ms) || ms <= 0) {
|
||||
return "0s";
|
||||
}
|
||||
const seconds = Math.round(ms / 1000);
|
||||
if (seconds < 60) {
|
||||
return `${seconds}s`;
|
||||
}
|
||||
const minutes = Math.round(seconds / 60);
|
||||
if (minutes < 60) {
|
||||
return `${minutes}m`;
|
||||
}
|
||||
const hours = Math.round(minutes / 60);
|
||||
if (hours < 48) {
|
||||
return `${hours}h`;
|
||||
}
|
||||
return `${Math.round(hours / 24)}d`;
|
||||
}
|
||||
|
||||
export function formatReviewList(items: SupervisorReviewItem[]): string {
|
||||
const rows = [
|
||||
[
|
||||
"id",
|
||||
"queuedAt",
|
||||
"reason",
|
||||
"chatroomId",
|
||||
"requestId",
|
||||
"target",
|
||||
"resolved",
|
||||
],
|
||||
...items.map((item) => [
|
||||
item.id,
|
||||
item.queuedAt,
|
||||
item.reason,
|
||||
item.chatroomId,
|
||||
item.requestId,
|
||||
item.target,
|
||||
item.resolution ? "yes" : "no",
|
||||
]),
|
||||
];
|
||||
return rows.map((row) => row.join("\t")).join("\n");
|
||||
}
|
||||
|
||||
export function formatReviewShow(item: SupervisorReviewItem): string {
|
||||
const packet = item.packet;
|
||||
const evidence =
|
||||
packet?.evidence
|
||||
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
|
||||
.join("\n") || "-";
|
||||
|
||||
return [
|
||||
`id: ${item.id}`,
|
||||
`queuedAt: ${item.queuedAt}`,
|
||||
`reason: ${item.reason}`,
|
||||
`target: ${item.target}`,
|
||||
`chatroomId: ${item.chatroomId}`,
|
||||
`userId: ${item.userId}`,
|
||||
`requestId: ${item.requestId}`,
|
||||
`sourceChain: ${packet?.sourceChain.join(" > ") || "-"}`,
|
||||
`userGoal: ${stringifyOptional(packet?.userGoal)}`,
|
||||
`summary: ${stringifyOptional(packet?.summary)}`,
|
||||
"[evidence]",
|
||||
evidence,
|
||||
item.resolution
|
||||
? [
|
||||
"[resolution]",
|
||||
`action: ${item.resolution.action}`,
|
||||
`resolvedAt: ${item.resolution.resolvedAt}`,
|
||||
`resolvedBy: ${item.resolution.resolvedBy}`,
|
||||
`note: ${stringifyOptional(item.resolution.note)}`,
|
||||
].join("\n")
|
||||
: "[resolution]\n-",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function formatReviewStats(stats: SupervisorReviewStats): string {
|
||||
const reasonRows = Object.entries(stats.byReason)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(
|
||||
([reason, counts]) =>
|
||||
`${reason}\ttotal=${counts.total}\tpending=${counts.pending}\tresolved=${counts.resolved}`,
|
||||
);
|
||||
const sourceRows = stats.topSourceChains.map(
|
||||
(item) => `${item.chain}\tcount=${item.count}`,
|
||||
);
|
||||
|
||||
return [
|
||||
`total=${stats.total} pending=${stats.pending} resolved=${stats.resolved} averagePendingAge=${formatAge(stats.averagePendingAgeMs)}`,
|
||||
"[byReason]",
|
||||
...(reasonRows.length > 0 ? reasonRows : ["-"]),
|
||||
"[topSourceChains]",
|
||||
...(sourceRows.length > 0 ? sourceRows : ["-"]),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function handleReviewList(
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
const reasonRaw = getFlagString(flags, "reason");
|
||||
if (reasonRaw && !isHandoffReason(reasonRaw)) {
|
||||
throw new Error(`Invalid --reason: ${reasonRaw}`);
|
||||
}
|
||||
const reason = isHandoffReason(reasonRaw) ? reasonRaw : undefined;
|
||||
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const items = await listSupervisorReview(connection.garnetClient, {
|
||||
includeResolved: hasFlag(flags, "include-resolved"),
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? 50,
|
||||
reason,
|
||||
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since")),
|
||||
untilQueuedAt: getFlagString(flags, "until"),
|
||||
});
|
||||
console.info(
|
||||
hasFlag(flags, "json")
|
||||
? JSON.stringify(items, null, 2)
|
||||
: formatReviewList(items),
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReviewShow(
|
||||
id: string | undefined,
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
if (!id?.trim()) {
|
||||
throw new Error("review:show requires an entry id");
|
||||
}
|
||||
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const item = await getSupervisorReview(connection.garnetClient, id);
|
||||
if (!item) {
|
||||
process.exitCode = 2;
|
||||
console.error(`[review] entry not found: ${id}`);
|
||||
return;
|
||||
}
|
||||
console.info(
|
||||
hasFlag(flags, "json")
|
||||
? JSON.stringify(item, null, 2)
|
||||
: formatReviewShow(item),
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReviewResolve(
|
||||
id: string | undefined,
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
if (!id?.trim()) {
|
||||
throw new Error("review:resolve requires an entry id");
|
||||
}
|
||||
const action = getFlagString(flags, "action");
|
||||
if (!isSupervisorReviewAction(action)) {
|
||||
throw new Error(
|
||||
"--action is required and must be manual-reply, reroute, or dismiss",
|
||||
);
|
||||
}
|
||||
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const item = await getSupervisorReview(connection.garnetClient, id);
|
||||
if (!item) {
|
||||
process.exitCode = 2;
|
||||
console.error(`[review] entry not found: ${id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
await resolveSupervisorReview(connection.garnetClient, id, {
|
||||
action,
|
||||
resolvedAt: new Date().toISOString(),
|
||||
resolvedBy: getFlagString(flags, "by") ?? getReviewerId(),
|
||||
...(getFlagString(flags, "note")
|
||||
? { note: getFlagString(flags, "note") }
|
||||
: {}),
|
||||
});
|
||||
console.info(`[review] resolved id=${id} action=${action}`);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReviewStats(
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const stats = await getSupervisorReviewStats(connection.garnetClient, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since") ?? "7d"),
|
||||
});
|
||||
console.info(
|
||||
hasFlag(flags, "json")
|
||||
? JSON.stringify(stats, null, 2)
|
||||
: formatReviewStats(stats),
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
loadEnvironmentFromNodeEnv();
|
||||
initLogger();
|
||||
|
||||
const { command, flags, positionals } = parseArgs(process.argv.slice(2));
|
||||
if (command === "review:list") {
|
||||
await handleReviewList(flags);
|
||||
return;
|
||||
}
|
||||
if (command === "review:show") {
|
||||
await handleReviewShow(positionals[0], flags);
|
||||
return;
|
||||
}
|
||||
if (command === "review:resolve") {
|
||||
await handleReviewResolve(positionals[0], flags);
|
||||
return;
|
||||
}
|
||||
if (command === "review:stats") {
|
||||
await handleReviewStats(flags);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"Unknown command. Use review:list, review:show, review:resolve, or review:stats.",
|
||||
);
|
||||
}
|
||||
|
||||
function isDirectCliEntry(): boolean {
|
||||
const entrypoint = process.argv[1];
|
||||
return entrypoint
|
||||
? import.meta.url === pathToFileURL(entrypoint).href
|
||||
: false;
|
||||
}
|
||||
|
||||
if (isDirectCliEntry()) {
|
||||
main().catch((error) => {
|
||||
process.exitCode = process.exitCode || 1;
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorReview,
|
||||
getSupervisorReviewStats,
|
||||
listSupervisorReview,
|
||||
parseSupervisorReviewEntry,
|
||||
resolveSupervisorReview,
|
||||
SUPERVISOR_RESOLVED_HASH_PREFIX,
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
} from "./review-queue.js";
|
||||
|
||||
class FakeGarnet {
|
||||
readonly hashes = new Map<string, Record<string, string>>();
|
||||
readonly stream: StreamEntry[];
|
||||
|
||||
constructor(stream: StreamEntry[]) {
|
||||
this.stream = stream;
|
||||
}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
|
||||
const entries = this.stream.filter((entry) => {
|
||||
const afterStart = start === "-" || entry.id >= start;
|
||||
const beforeEnd = end === "+" || entry.id <= end;
|
||||
return afterStart && beforeEnd;
|
||||
});
|
||||
return Promise.resolve(options?.count ? entries.slice(0, options.count) : entries);
|
||||
}
|
||||
|
||||
getHash(key: string): Promise<Record<string, string>> {
|
||||
return Promise.resolve({ ...(this.hashes.get(key) ?? {}) });
|
||||
}
|
||||
|
||||
setHash(key: string, fields: Record<string, string>): Promise<void> {
|
||||
this.hashes.set(key, { ...fields });
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
expireKey(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
}
|
||||
|
||||
const packet = {
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "out-of-scope",
|
||||
sourceChain: ["onboarding", "moderation"],
|
||||
summary: "user asked about ads",
|
||||
userGoal: "광고 규칙 문의",
|
||||
};
|
||||
|
||||
const entries: StreamEntry[] = [
|
||||
{
|
||||
id: "1-0",
|
||||
message: {
|
||||
chatroomId: "room-1",
|
||||
packet: JSON.stringify(packet),
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "out-of-scope",
|
||||
requestId: "request-1",
|
||||
target: "moderation",
|
||||
type: "supervisor-handoff",
|
||||
userId: "user-1",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "2-0",
|
||||
message: {
|
||||
chatroomId: "room-2",
|
||||
queuedAt: "2026-05-18T01:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-2",
|
||||
target: "human-review",
|
||||
type: "supervisor-handoff",
|
||||
userId: "user-2",
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe("supervisor review queue", () => {
|
||||
const firstEntry = entries[0]!;
|
||||
|
||||
it("parses a stream entry with the preserved handoff packet", () => {
|
||||
expect(parseSupervisorReviewEntry(firstEntry)).toMatchObject({
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
packet,
|
||||
reason: "out-of-scope",
|
||||
target: "moderation",
|
||||
});
|
||||
});
|
||||
|
||||
it("lists pending entries and filters resolved items by default", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
await resolveSupervisorReview(garnet, "1-0", {
|
||||
action: "manual-reply",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
|
||||
await expect(listSupervisorReview(garnet)).resolves.toHaveLength(1);
|
||||
await expect(
|
||||
listSupervisorReview(garnet, { includeResolved: true, reason: "out-of-scope" }),
|
||||
).resolves.toMatchObject([
|
||||
{
|
||||
id: "1-0",
|
||||
resolution: {
|
||||
action: "manual-reply",
|
||||
resolvedBy: "operator@example.com",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("gets a single entry with resolution metadata", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
||||
action: "dismiss",
|
||||
note: "duplicate",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
|
||||
await expect(getSupervisorReview(garnet, "1-0")).resolves.toMatchObject({
|
||||
id: "1-0",
|
||||
resolution: {
|
||||
action: "dismiss",
|
||||
note: "duplicate",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("computes reason totals and top source chains", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
||||
action: "reroute",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
|
||||
await expect(getSupervisorReviewStats(garnet)).resolves.toMatchObject({
|
||||
byReason: {
|
||||
"low-confidence": {
|
||||
pending: 1,
|
||||
resolved: 0,
|
||||
total: 1,
|
||||
},
|
||||
"out-of-scope": {
|
||||
pending: 0,
|
||||
resolved: 1,
|
||||
total: 1,
|
||||
},
|
||||
},
|
||||
pending: 1,
|
||||
resolved: 1,
|
||||
topSourceChains: [{ chain: "onboarding>moderation", count: 1 }],
|
||||
total: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,14 +1,344 @@
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorReviewQueueMaxLen,
|
||||
getSupervisorReviewResolutionTtlSec,
|
||||
} from "../../config.js";
|
||||
|
||||
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
|
||||
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
|
||||
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
|
||||
"lolympus-agent:review-resolved:";
|
||||
|
||||
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
|
||||
|
||||
export interface SupervisorReviewEntry {
|
||||
id: string;
|
||||
queuedAt: string;
|
||||
chatroomId: string;
|
||||
userId: string;
|
||||
requestId: string;
|
||||
reason: HandoffReason;
|
||||
target: string;
|
||||
packet?: HandoffPacket | undefined;
|
||||
extra?: Record<string, string> | undefined;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewResolution {
|
||||
resolvedAt: string;
|
||||
resolvedBy: string;
|
||||
action: SupervisorReviewAction;
|
||||
note?: string | undefined;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewItem extends SupervisorReviewEntry {
|
||||
resolution?: SupervisorReviewResolution | undefined;
|
||||
}
|
||||
|
||||
export interface ListSupervisorReviewOptions {
|
||||
includeResolved?: boolean | undefined;
|
||||
limit?: number | undefined;
|
||||
reason?: HandoffReason | undefined;
|
||||
since?: string | undefined;
|
||||
sinceQueuedAt?: string | undefined;
|
||||
until?: string | undefined;
|
||||
untilQueuedAt?: string | undefined;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewReasonStats {
|
||||
pending: number;
|
||||
resolved: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SupervisorReviewStats {
|
||||
averagePendingAgeMs: number;
|
||||
byReason: Record<string, SupervisorReviewReasonStats>;
|
||||
pending: number;
|
||||
resolved: number;
|
||||
topSourceChains: Array<{ chain: string; count: number }>;
|
||||
total: number;
|
||||
}
|
||||
|
||||
type SupervisorReviewListGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"getHash" | "readStreamRange"
|
||||
>;
|
||||
|
||||
type SupervisorReviewResolveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"expireKey" | "setHash"
|
||||
>;
|
||||
|
||||
const BASE_FIELDS = new Set([
|
||||
"chatroomId",
|
||||
"packet",
|
||||
"queuedAt",
|
||||
"reason",
|
||||
"requestId",
|
||||
"target",
|
||||
"type",
|
||||
"userId",
|
||||
]);
|
||||
|
||||
function isHandoffReason(value: string | undefined): value is HandoffReason {
|
||||
return (
|
||||
value === "out-of-scope" ||
|
||||
value === "missing-tool" ||
|
||||
value === "low-confidence" ||
|
||||
value === "validation-failed" ||
|
||||
value === "same-agent-revisit" ||
|
||||
value === "max-hop" ||
|
||||
value === "high-risk-action" ||
|
||||
value === "policy-risk"
|
||||
);
|
||||
}
|
||||
|
||||
function isSupervisorReviewAction(
|
||||
value: string | undefined,
|
||||
): value is SupervisorReviewAction {
|
||||
return value === "manual-reply" || value === "reroute" || value === "dismiss";
|
||||
}
|
||||
|
||||
function parseOptionalPacket(value: string | undefined): HandoffPacket | undefined {
|
||||
if (!value?.trim() || value.trim() === "null") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
return typeof parsed === "object" && parsed !== null
|
||||
? (parsed as HandoffPacket)
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function parseResolution(
|
||||
fields: Record<string, string>,
|
||||
): SupervisorReviewResolution | undefined {
|
||||
if (
|
||||
!fields["resolvedAt"] ||
|
||||
!fields["resolvedBy"] ||
|
||||
!isSupervisorReviewAction(fields["action"])
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
action: fields["action"],
|
||||
resolvedAt: fields["resolvedAt"],
|
||||
resolvedBy: fields["resolvedBy"],
|
||||
...(fields["note"] ? { note: fields["note"] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolutionKey(id: string): string {
|
||||
return `${SUPERVISOR_RESOLVED_HASH_PREFIX}${id}`;
|
||||
}
|
||||
|
||||
function isAfterOrEqual(value: string, lowerBound: string | undefined): boolean {
|
||||
return !lowerBound || value >= lowerBound;
|
||||
}
|
||||
|
||||
function isBeforeOrEqual(value: string, upperBound: string | undefined): boolean {
|
||||
return !upperBound || value <= upperBound;
|
||||
}
|
||||
|
||||
export async function appendSupervisorReview(
|
||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
await garnetClient.appendStream(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
{
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
},
|
||||
{ maxLen: getSupervisorReviewQueueMaxLen() },
|
||||
);
|
||||
}
|
||||
|
||||
export function parseSupervisorReviewEntry(
|
||||
streamEntry: StreamEntry,
|
||||
): SupervisorReviewEntry | null {
|
||||
const { message } = streamEntry;
|
||||
const reason = message["reason"];
|
||||
|
||||
if (
|
||||
!message["queuedAt"] ||
|
||||
!message["chatroomId"] ||
|
||||
!message["userId"] ||
|
||||
!message["requestId"] ||
|
||||
!message["target"] ||
|
||||
!isHandoffReason(reason)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const extra = Object.fromEntries(
|
||||
Object.entries(message).filter(([key]) => !BASE_FIELDS.has(key)),
|
||||
);
|
||||
const packet = parseOptionalPacket(message["packet"]);
|
||||
|
||||
return {
|
||||
chatroomId: message["chatroomId"],
|
||||
id: streamEntry.id,
|
||||
queuedAt: message["queuedAt"],
|
||||
reason,
|
||||
requestId: message["requestId"],
|
||||
target: message["target"],
|
||||
userId: message["userId"],
|
||||
...(packet ? { packet } : {}),
|
||||
...(Object.keys(extra).length > 0 ? { extra } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function getSupervisorReview(
|
||||
garnetClient: SupervisorReviewListGarnetClient,
|
||||
id: string,
|
||||
): Promise<SupervisorReviewItem | null> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
id,
|
||||
id,
|
||||
{ count: 1 },
|
||||
);
|
||||
const parsed = entries.flatMap((entry) => {
|
||||
const review = parseSupervisorReviewEntry(entry);
|
||||
return review ? [review] : [];
|
||||
})[0];
|
||||
|
||||
if (!parsed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const resolution = parseResolution(
|
||||
await garnetClient.getHash(resolutionKey(parsed.id)),
|
||||
);
|
||||
return {
|
||||
...parsed,
|
||||
...(resolution ? { resolution } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listSupervisorReview(
|
||||
garnetClient: SupervisorReviewListGarnetClient,
|
||||
options?: ListSupervisorReviewOptions,
|
||||
): Promise<SupervisorReviewItem[]> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
options?.since ?? "-",
|
||||
options?.until ?? "+",
|
||||
);
|
||||
const items: SupervisorReviewItem[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
const review = parseSupervisorReviewEntry(entry);
|
||||
if (!review) {
|
||||
continue;
|
||||
}
|
||||
if (options?.reason && review.reason !== options.reason) {
|
||||
continue;
|
||||
}
|
||||
if (!isAfterOrEqual(review.queuedAt, options?.sinceQueuedAt)) {
|
||||
continue;
|
||||
}
|
||||
if (!isBeforeOrEqual(review.queuedAt, options?.untilQueuedAt)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const resolution = parseResolution(
|
||||
await garnetClient.getHash(resolutionKey(review.id)),
|
||||
);
|
||||
if (resolution && !options?.includeResolved) {
|
||||
continue;
|
||||
}
|
||||
items.push({
|
||||
...review,
|
||||
...(resolution ? { resolution } : {}),
|
||||
});
|
||||
if (options?.limit && items.length >= options.limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
export async function resolveSupervisorReview(
|
||||
garnetClient: SupervisorReviewResolveGarnetClient,
|
||||
id: string,
|
||||
resolution: SupervisorReviewResolution,
|
||||
): Promise<void> {
|
||||
const fields: Record<string, string> = {
|
||||
action: resolution.action,
|
||||
resolvedAt: resolution.resolvedAt,
|
||||
resolvedBy: resolution.resolvedBy,
|
||||
};
|
||||
if (resolution.note?.trim()) {
|
||||
fields.note = resolution.note.trim();
|
||||
}
|
||||
|
||||
const key = resolutionKey(id);
|
||||
await garnetClient.setHash(key, fields);
|
||||
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
|
||||
}
|
||||
|
||||
export async function getSupervisorReviewStats(
|
||||
garnetClient: SupervisorReviewListGarnetClient,
|
||||
options?: Omit<ListSupervisorReviewOptions, "includeResolved">,
|
||||
): Promise<SupervisorReviewStats> {
|
||||
const nowMs = Date.now();
|
||||
const items = await listSupervisorReview(garnetClient, {
|
||||
...options,
|
||||
includeResolved: true,
|
||||
});
|
||||
const byReason: Record<string, SupervisorReviewReasonStats> = {};
|
||||
const sourceChains = new Map<string, number>();
|
||||
let pendingAgeMs = 0;
|
||||
let pending = 0;
|
||||
let resolved = 0;
|
||||
|
||||
for (const item of items) {
|
||||
const reasonStats = byReason[item.reason] ?? {
|
||||
pending: 0,
|
||||
resolved: 0,
|
||||
total: 0,
|
||||
};
|
||||
reasonStats.total += 1;
|
||||
|
||||
if (item.resolution) {
|
||||
resolved += 1;
|
||||
reasonStats.resolved += 1;
|
||||
} else {
|
||||
pending += 1;
|
||||
reasonStats.pending += 1;
|
||||
const queuedAtMs = Date.parse(item.queuedAt);
|
||||
if (Number.isFinite(queuedAtMs)) {
|
||||
pendingAgeMs += Math.max(0, nowMs - queuedAtMs);
|
||||
}
|
||||
}
|
||||
|
||||
byReason[item.reason] = reasonStats;
|
||||
|
||||
const chain = item.packet?.sourceChain?.join(">");
|
||||
if (chain) {
|
||||
sourceChains.set(chain, (sourceChains.get(chain) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
averagePendingAgeMs: pending === 0 ? 0 : Math.round(pendingAgeMs / pending),
|
||||
byReason,
|
||||
pending,
|
||||
resolved,
|
||||
topSourceChains: [...sourceChains.entries()]
|
||||
.map(([chain, count]) => ({ chain, count }))
|
||||
.sort((left, right) => right.count - left.count)
|
||||
.slice(0, 5),
|
||||
total: items.length,
|
||||
};
|
||||
}
|
||||
@@ -82,6 +82,15 @@ describe("SupervisorSkill", () => {
|
||||
type: "supervisor-handoff",
|
||||
},
|
||||
});
|
||||
expect(garnet.streams[0]?.fields["packet"]).toEqual(expect.any(String));
|
||||
const queuedPacket: unknown = JSON.parse(
|
||||
String(garnet.streams[0]?.fields["packet"]),
|
||||
);
|
||||
expect(queuedPacket).toMatchObject({
|
||||
handoffReason: "max-hop",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
userGoal: "가입 상태 확인",
|
||||
});
|
||||
expect(nats.published[0]).toMatchObject({
|
||||
kind: "supervisor-handoff",
|
||||
metric: "agent.supervisor_handoff",
|
||||
|
||||
@@ -56,6 +56,7 @@ export class SupervisorSkill implements Skill {
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
chatroomId: request.chatroomId,
|
||||
packet: JSON.stringify(packet ?? null),
|
||||
reason,
|
||||
requestId: request.requestId,
|
||||
target: target ?? "human-review",
|
||||
|
||||
Reference in new issue
Block a user