Close supervisor review loop and add handoff eval

This commit is contained in:
dahyeon committed 2026-05-18 18:04:01 -07:00
1 parent c14adac162
commit f0b00e5fd2
20 files changed
+2017 -129

No files matched your search

+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import {
formatReviewList,
formatReviewShow,
formatReviewStats,
} from "./cli.js";
describe("operations review CLI formatters", () => {
it("formats review list rows as tab-separated output", () => {
expect(
formatReviewList([
{
chatroomId: "room-1",
id: "1-0",
queuedAt: "2026-05-18T00:00:00.000Z",
reason: "low-confidence",
requestId: "request-1",
target: "human-review",
userId: "user-1",
},
]),
).toContain("1-0\t2026-05-18T00:00:00.000Z\tlow-confidence");
});
it("formats packet details for show output", () => {
expect(
formatReviewShow({
chatroomId: "room-1",
id: "1-0",
packet: {
constraints: [],
currentState: {},
evidence: [
{
id: "e1",
kind: "message",
summary: "unclear message",
},
],
expectedOutputSchema: "text",
handoffReason: "low-confidence",
sourceChain: ["onboarding", "supervisor"],
summary: "needs review",
userGoal: "의도 확인",
},
queuedAt: "2026-05-18T00:00:00.000Z",
reason: "low-confidence",
requestId: "request-1",
target: "human-review",
userId: "user-1",
}),
).toContain("sourceChain: onboarding > supervisor");
});
it("formats aggregate stats", () => {
expect(
formatReviewStats({
averagePendingAgeMs: 60_000,
byReason: {
"low-confidence": {
pending: 1,
resolved: 2,
total: 3,
},
},
pending: 1,
resolved: 2,
topSourceChains: [{ chain: "onboarding>supervisor", count: 3 }],
total: 3,
}),
).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2");
});
});
+299
View File
@@ -0,0 +1,299 @@
import { pathToFileURL } from "node:url";
import type { HandoffReason } from "@agent/types";
import { loadEnvironmentFromNodeEnv } from "@common/env-loader";
import { connectGarnetForCli } from "@common/garnet-cli";
import { initLogger } from "@obs/logger";
import {
getSupervisorReview,
getSupervisorReviewStats,
listSupervisorReview,
resolveSupervisorReview,
type SupervisorReviewAction,
type SupervisorReviewItem,
type SupervisorReviewStats,
} from "@skills/supervisor/review-queue";
import {
getFlagString,
hasFlag,
parseArgs,
parseDurationMs,
parsePositiveInteger,
} from "../eval/cli-args.js";
import { getReviewerId } from "../eval/review/cli.js";
function isHandoffReason(value: string | undefined): value is HandoffReason {
return (
value === "out-of-scope" ||
value === "missing-tool" ||
value === "low-confidence" ||
value === "validation-failed" ||
value === "same-agent-revisit" ||
value === "max-hop" ||
value === "high-risk-action" ||
value === "policy-risk"
);
}
function isSupervisorReviewAction(
value: string | undefined,
): value is SupervisorReviewAction {
return value === "manual-reply" || value === "reroute" || value === "dismiss";
}
function sinceDurationToIso(value: string | undefined): string | undefined {
if (!value?.trim()) {
return undefined;
}
return new Date(Date.now() - parseDurationMs(value)).toISOString();
}
function stringifyOptional(value: string | undefined): string {
return value?.trim() ? value : "-";
}
function formatAge(ms: number): string {
if (!Number.isFinite(ms) || ms <= 0) {
return "0s";
}
const seconds = Math.round(ms / 1000);
if (seconds < 60) {
return `${seconds}s`;
}
const minutes = Math.round(seconds / 60);
if (minutes < 60) {
return `${minutes}m`;
}
const hours = Math.round(minutes / 60);
if (hours < 48) {
return `${hours}h`;
}
return `${Math.round(hours / 24)}d`;
}
export function formatReviewList(items: SupervisorReviewItem[]): string {
const rows = [
[
"id",
"queuedAt",
"reason",
"chatroomId",
"requestId",
"target",
"resolved",
],
...items.map((item) => [
item.id,
item.queuedAt,
item.reason,
item.chatroomId,
item.requestId,
item.target,
item.resolution ? "yes" : "no",
]),
];
return rows.map((row) => row.join("\t")).join("\n");
}
export function formatReviewShow(item: SupervisorReviewItem): string {
const packet = item.packet;
const evidence =
packet?.evidence
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
.join("\n") || "-";
return [
`id: ${item.id}`,
`queuedAt: ${item.queuedAt}`,
`reason: ${item.reason}`,
`target: ${item.target}`,
`chatroomId: ${item.chatroomId}`,
`userId: ${item.userId}`,
`requestId: ${item.requestId}`,
`sourceChain: ${packet?.sourceChain.join(" > ") || "-"}`,
`userGoal: ${stringifyOptional(packet?.userGoal)}`,
`summary: ${stringifyOptional(packet?.summary)}`,
"[evidence]",
evidence,
item.resolution
? [
"[resolution]",
`action: ${item.resolution.action}`,
`resolvedAt: ${item.resolution.resolvedAt}`,
`resolvedBy: ${item.resolution.resolvedBy}`,
`note: ${stringifyOptional(item.resolution.note)}`,
].join("\n")
: "[resolution]\n-",
].join("\n");
}
export function formatReviewStats(stats: SupervisorReviewStats): string {
const reasonRows = Object.entries(stats.byReason)
.sort(([left], [right]) => left.localeCompare(right))
.map(
([reason, counts]) =>
`${reason}\ttotal=${counts.total}\tpending=${counts.pending}\tresolved=${counts.resolved}`,
);
const sourceRows = stats.topSourceChains.map(
(item) => `${item.chain}\tcount=${item.count}`,
);
return [
`total=${stats.total} pending=${stats.pending} resolved=${stats.resolved} averagePendingAge=${formatAge(stats.averagePendingAgeMs)}`,
"[byReason]",
...(reasonRows.length > 0 ? reasonRows : ["-"]),
"[topSourceChains]",
...(sourceRows.length > 0 ? sourceRows : ["-"]),
].join("\n");
}
async function handleReviewList(
flags: Map<string, string | true>,
): Promise<void> {
const reasonRaw = getFlagString(flags, "reason");
if (reasonRaw && !isHandoffReason(reasonRaw)) {
throw new Error(`Invalid --reason: ${reasonRaw}`);
}
const reason = isHandoffReason(reasonRaw) ? reasonRaw : undefined;
const connection = await connectGarnetForCli();
try {
const items = await listSupervisorReview(connection.garnetClient, {
includeResolved: hasFlag(flags, "include-resolved"),
limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? 50,
reason,
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since")),
untilQueuedAt: getFlagString(flags, "until"),
});
console.info(
hasFlag(flags, "json")
? JSON.stringify(items, null, 2)
: formatReviewList(items),
);
} finally {
await connection.close();
}
}
async function handleReviewShow(
id: string | undefined,
flags: Map<string, string | true>,
): Promise<void> {
if (!id?.trim()) {
throw new Error("review:show requires an entry id");
}
const connection = await connectGarnetForCli();
try {
const item = await getSupervisorReview(connection.garnetClient, id);
if (!item) {
process.exitCode = 2;
console.error(`[review] entry not found: ${id}`);
return;
}
console.info(
hasFlag(flags, "json")
? JSON.stringify(item, null, 2)
: formatReviewShow(item),
);
} finally {
await connection.close();
}
}
async function handleReviewResolve(
id: string | undefined,
flags: Map<string, string | true>,
): Promise<void> {
if (!id?.trim()) {
throw new Error("review:resolve requires an entry id");
}
const action = getFlagString(flags, "action");
if (!isSupervisorReviewAction(action)) {
throw new Error(
"--action is required and must be manual-reply, reroute, or dismiss",
);
}
const connection = await connectGarnetForCli();
try {
const item = await getSupervisorReview(connection.garnetClient, id);
if (!item) {
process.exitCode = 2;
console.error(`[review] entry not found: ${id}`);
return;
}
await resolveSupervisorReview(connection.garnetClient, id, {
action,
resolvedAt: new Date().toISOString(),
resolvedBy: getFlagString(flags, "by") ?? getReviewerId(),
...(getFlagString(flags, "note")
? { note: getFlagString(flags, "note") }
: {}),
});
console.info(`[review] resolved id=${id} action=${action}`);
} finally {
await connection.close();
}
}
async function handleReviewStats(
flags: Map<string, string | true>,
): Promise<void> {
const connection = await connectGarnetForCli();
try {
const stats = await getSupervisorReviewStats(connection.garnetClient, {
limit: parsePositiveInteger(getFlagString(flags, "limit")),
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since") ?? "7d"),
});
console.info(
hasFlag(flags, "json")
? JSON.stringify(stats, null, 2)
: formatReviewStats(stats),
);
} finally {
await connection.close();
}
}
async function main(): Promise<void> {
loadEnvironmentFromNodeEnv();
initLogger();
const { command, flags, positionals } = parseArgs(process.argv.slice(2));
if (command === "review:list") {
await handleReviewList(flags);
return;
}
if (command === "review:show") {
await handleReviewShow(positionals[0], flags);
return;
}
if (command === "review:resolve") {
await handleReviewResolve(positionals[0], flags);
return;
}
if (command === "review:stats") {
await handleReviewStats(flags);
return;
}
throw new Error(
"Unknown command. Use review:list, review:show, review:resolve, or review:stats.",
);
}
function isDirectCliEntry(): boolean {
const entrypoint = process.argv[1];
return entrypoint
? import.meta.url === pathToFileURL(entrypoint).href
: false;
}
if (isDirectCliEntry()) {
main().catch((error) => {
process.exitCode = process.exitCode || 1;
console.error(error instanceof Error ? error.message : String(error));
});
}