Close supervisor review loop and add handoff eval
This commit is contained in:
1 parent
c14adac162
commit
f0b00e5fd2
20 files changed
+2017
-129
No files matched your search
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
formatReviewList,
|
||||
formatReviewShow,
|
||||
formatReviewStats,
|
||||
} from "./cli.js";
|
||||
|
||||
describe("operations review CLI formatters", () => {
|
||||
it("formats review list rows as tab-separated output", () => {
|
||||
expect(
|
||||
formatReviewList([
|
||||
{
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
userId: "user-1",
|
||||
},
|
||||
]),
|
||||
).toContain("1-0\t2026-05-18T00:00:00.000Z\tlow-confidence");
|
||||
});
|
||||
|
||||
it("formats packet details for show output", () => {
|
||||
expect(
|
||||
formatReviewShow({
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
packet: {
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [
|
||||
{
|
||||
id: "e1",
|
||||
kind: "message",
|
||||
summary: "unclear message",
|
||||
},
|
||||
],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "low-confidence",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
summary: "needs review",
|
||||
userGoal: "의도 확인",
|
||||
},
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
userId: "user-1",
|
||||
}),
|
||||
).toContain("sourceChain: onboarding > supervisor");
|
||||
});
|
||||
|
||||
it("formats aggregate stats", () => {
|
||||
expect(
|
||||
formatReviewStats({
|
||||
averagePendingAgeMs: 60_000,
|
||||
byReason: {
|
||||
"low-confidence": {
|
||||
pending: 1,
|
||||
resolved: 2,
|
||||
total: 3,
|
||||
},
|
||||
},
|
||||
pending: 1,
|
||||
resolved: 2,
|
||||
topSourceChains: [{ chain: "onboarding>supervisor", count: 3 }],
|
||||
total: 3,
|
||||
}),
|
||||
).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import { loadEnvironmentFromNodeEnv } from "@common/env-loader";
|
||||
import { connectGarnetForCli } from "@common/garnet-cli";
|
||||
import { initLogger } from "@obs/logger";
|
||||
import {
|
||||
getSupervisorReview,
|
||||
getSupervisorReviewStats,
|
||||
listSupervisorReview,
|
||||
resolveSupervisorReview,
|
||||
type SupervisorReviewAction,
|
||||
type SupervisorReviewItem,
|
||||
type SupervisorReviewStats,
|
||||
} from "@skills/supervisor/review-queue";
|
||||
import {
|
||||
getFlagString,
|
||||
hasFlag,
|
||||
parseArgs,
|
||||
parseDurationMs,
|
||||
parsePositiveInteger,
|
||||
} from "../eval/cli-args.js";
|
||||
import { getReviewerId } from "../eval/review/cli.js";
|
||||
|
||||
function isHandoffReason(value: string | undefined): value is HandoffReason {
|
||||
return (
|
||||
value === "out-of-scope" ||
|
||||
value === "missing-tool" ||
|
||||
value === "low-confidence" ||
|
||||
value === "validation-failed" ||
|
||||
value === "same-agent-revisit" ||
|
||||
value === "max-hop" ||
|
||||
value === "high-risk-action" ||
|
||||
value === "policy-risk"
|
||||
);
|
||||
}
|
||||
|
||||
function isSupervisorReviewAction(
|
||||
value: string | undefined,
|
||||
): value is SupervisorReviewAction {
|
||||
return value === "manual-reply" || value === "reroute" || value === "dismiss";
|
||||
}
|
||||
|
||||
function sinceDurationToIso(value: string | undefined): string | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
return new Date(Date.now() - parseDurationMs(value)).toISOString();
|
||||
}
|
||||
|
||||
function stringifyOptional(value: string | undefined): string {
|
||||
return value?.trim() ? value : "-";
|
||||
}
|
||||
|
||||
function formatAge(ms: number): string {
|
||||
if (!Number.isFinite(ms) || ms <= 0) {
|
||||
return "0s";
|
||||
}
|
||||
const seconds = Math.round(ms / 1000);
|
||||
if (seconds < 60) {
|
||||
return `${seconds}s`;
|
||||
}
|
||||
const minutes = Math.round(seconds / 60);
|
||||
if (minutes < 60) {
|
||||
return `${minutes}m`;
|
||||
}
|
||||
const hours = Math.round(minutes / 60);
|
||||
if (hours < 48) {
|
||||
return `${hours}h`;
|
||||
}
|
||||
return `${Math.round(hours / 24)}d`;
|
||||
}
|
||||
|
||||
export function formatReviewList(items: SupervisorReviewItem[]): string {
|
||||
const rows = [
|
||||
[
|
||||
"id",
|
||||
"queuedAt",
|
||||
"reason",
|
||||
"chatroomId",
|
||||
"requestId",
|
||||
"target",
|
||||
"resolved",
|
||||
],
|
||||
...items.map((item) => [
|
||||
item.id,
|
||||
item.queuedAt,
|
||||
item.reason,
|
||||
item.chatroomId,
|
||||
item.requestId,
|
||||
item.target,
|
||||
item.resolution ? "yes" : "no",
|
||||
]),
|
||||
];
|
||||
return rows.map((row) => row.join("\t")).join("\n");
|
||||
}
|
||||
|
||||
export function formatReviewShow(item: SupervisorReviewItem): string {
|
||||
const packet = item.packet;
|
||||
const evidence =
|
||||
packet?.evidence
|
||||
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
|
||||
.join("\n") || "-";
|
||||
|
||||
return [
|
||||
`id: ${item.id}`,
|
||||
`queuedAt: ${item.queuedAt}`,
|
||||
`reason: ${item.reason}`,
|
||||
`target: ${item.target}`,
|
||||
`chatroomId: ${item.chatroomId}`,
|
||||
`userId: ${item.userId}`,
|
||||
`requestId: ${item.requestId}`,
|
||||
`sourceChain: ${packet?.sourceChain.join(" > ") || "-"}`,
|
||||
`userGoal: ${stringifyOptional(packet?.userGoal)}`,
|
||||
`summary: ${stringifyOptional(packet?.summary)}`,
|
||||
"[evidence]",
|
||||
evidence,
|
||||
item.resolution
|
||||
? [
|
||||
"[resolution]",
|
||||
`action: ${item.resolution.action}`,
|
||||
`resolvedAt: ${item.resolution.resolvedAt}`,
|
||||
`resolvedBy: ${item.resolution.resolvedBy}`,
|
||||
`note: ${stringifyOptional(item.resolution.note)}`,
|
||||
].join("\n")
|
||||
: "[resolution]\n-",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function formatReviewStats(stats: SupervisorReviewStats): string {
|
||||
const reasonRows = Object.entries(stats.byReason)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(
|
||||
([reason, counts]) =>
|
||||
`${reason}\ttotal=${counts.total}\tpending=${counts.pending}\tresolved=${counts.resolved}`,
|
||||
);
|
||||
const sourceRows = stats.topSourceChains.map(
|
||||
(item) => `${item.chain}\tcount=${item.count}`,
|
||||
);
|
||||
|
||||
return [
|
||||
`total=${stats.total} pending=${stats.pending} resolved=${stats.resolved} averagePendingAge=${formatAge(stats.averagePendingAgeMs)}`,
|
||||
"[byReason]",
|
||||
...(reasonRows.length > 0 ? reasonRows : ["-"]),
|
||||
"[topSourceChains]",
|
||||
...(sourceRows.length > 0 ? sourceRows : ["-"]),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function handleReviewList(
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
const reasonRaw = getFlagString(flags, "reason");
|
||||
if (reasonRaw && !isHandoffReason(reasonRaw)) {
|
||||
throw new Error(`Invalid --reason: ${reasonRaw}`);
|
||||
}
|
||||
const reason = isHandoffReason(reasonRaw) ? reasonRaw : undefined;
|
||||
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const items = await listSupervisorReview(connection.garnetClient, {
|
||||
includeResolved: hasFlag(flags, "include-resolved"),
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? 50,
|
||||
reason,
|
||||
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since")),
|
||||
untilQueuedAt: getFlagString(flags, "until"),
|
||||
});
|
||||
console.info(
|
||||
hasFlag(flags, "json")
|
||||
? JSON.stringify(items, null, 2)
|
||||
: formatReviewList(items),
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReviewShow(
|
||||
id: string | undefined,
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
if (!id?.trim()) {
|
||||
throw new Error("review:show requires an entry id");
|
||||
}
|
||||
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const item = await getSupervisorReview(connection.garnetClient, id);
|
||||
if (!item) {
|
||||
process.exitCode = 2;
|
||||
console.error(`[review] entry not found: ${id}`);
|
||||
return;
|
||||
}
|
||||
console.info(
|
||||
hasFlag(flags, "json")
|
||||
? JSON.stringify(item, null, 2)
|
||||
: formatReviewShow(item),
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReviewResolve(
|
||||
id: string | undefined,
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
if (!id?.trim()) {
|
||||
throw new Error("review:resolve requires an entry id");
|
||||
}
|
||||
const action = getFlagString(flags, "action");
|
||||
if (!isSupervisorReviewAction(action)) {
|
||||
throw new Error(
|
||||
"--action is required and must be manual-reply, reroute, or dismiss",
|
||||
);
|
||||
}
|
||||
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const item = await getSupervisorReview(connection.garnetClient, id);
|
||||
if (!item) {
|
||||
process.exitCode = 2;
|
||||
console.error(`[review] entry not found: ${id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
await resolveSupervisorReview(connection.garnetClient, id, {
|
||||
action,
|
||||
resolvedAt: new Date().toISOString(),
|
||||
resolvedBy: getFlagString(flags, "by") ?? getReviewerId(),
|
||||
...(getFlagString(flags, "note")
|
||||
? { note: getFlagString(flags, "note") }
|
||||
: {}),
|
||||
});
|
||||
console.info(`[review] resolved id=${id} action=${action}`);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReviewStats(
|
||||
flags: Map<string, string | true>,
|
||||
): Promise<void> {
|
||||
const connection = await connectGarnetForCli();
|
||||
try {
|
||||
const stats = await getSupervisorReviewStats(connection.garnetClient, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
sinceQueuedAt: sinceDurationToIso(getFlagString(flags, "since") ?? "7d"),
|
||||
});
|
||||
console.info(
|
||||
hasFlag(flags, "json")
|
||||
? JSON.stringify(stats, null, 2)
|
||||
: formatReviewStats(stats),
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
loadEnvironmentFromNodeEnv();
|
||||
initLogger();
|
||||
|
||||
const { command, flags, positionals } = parseArgs(process.argv.slice(2));
|
||||
if (command === "review:list") {
|
||||
await handleReviewList(flags);
|
||||
return;
|
||||
}
|
||||
if (command === "review:show") {
|
||||
await handleReviewShow(positionals[0], flags);
|
||||
return;
|
||||
}
|
||||
if (command === "review:resolve") {
|
||||
await handleReviewResolve(positionals[0], flags);
|
||||
return;
|
||||
}
|
||||
if (command === "review:stats") {
|
||||
await handleReviewStats(flags);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"Unknown command. Use review:list, review:show, review:resolve, or review:stats.",
|
||||
);
|
||||
}
|
||||
|
||||
function isDirectCliEntry(): boolean {
|
||||
const entrypoint = process.argv[1];
|
||||
return entrypoint
|
||||
? import.meta.url === pathToFileURL(entrypoint).href
|
||||
: false;
|
||||
}
|
||||
|
||||
if (isDirectCliEntry()) {
|
||||
main().catch((error) => {
|
||||
process.exitCode = process.exitCode || 1;
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user