Close multi-agent plan and add router shadow reporting
This commit is contained in:
1 parent
f1b30d716d
commit
8156e8c07e
15 files changed
+1303
-204
No files matched your search
@@ -7,6 +7,7 @@ import {
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getSlackAlertWebhookUrl,
|
||||
getSlackWebhookTimeoutMs,
|
||||
getSupervisorLlmResponseEnabled,
|
||||
} from "./config.js";
|
||||
|
||||
const ENV_KEYS = [
|
||||
@@ -20,6 +21,7 @@ const ENV_KEYS = [
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
||||
"SLACK_ALERT_WEBHOOK_URL",
|
||||
"SLACK_WEBHOOK_TIMEOUT_MS",
|
||||
"AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED",
|
||||
] as const;
|
||||
|
||||
describe("config", () => {
|
||||
@@ -59,4 +61,12 @@ describe("config", () => {
|
||||
expect(getSlackAlertWebhookUrl()).toBe("https://hooks.slack.test/x");
|
||||
expect(getSlackWebhookTimeoutMs()).toBe(1234);
|
||||
});
|
||||
|
||||
it("enables supervisor LLM user messages by default with env override", () => {
|
||||
expect(getSupervisorLlmResponseEnabled()).toBe(true);
|
||||
|
||||
vi.stubEnv("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", "false");
|
||||
|
||||
expect(getSupervisorLlmResponseEnabled()).toBe(false);
|
||||
});
|
||||
});
|
||||
+23
-21
@@ -7,20 +7,14 @@ type LogLevel =
|
||||
| "trace"
|
||||
| "silent";
|
||||
|
||||
function getPositiveNumber(
|
||||
envKey: string,
|
||||
fallback: number,
|
||||
): number {
|
||||
function getPositiveNumber(envKey: string, fallback: number): number {
|
||||
const raw = process.env[envKey]?.trim();
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function getRatioNumber(
|
||||
envKey: string,
|
||||
fallback: number,
|
||||
): number {
|
||||
function getRatioNumber(envKey: string, fallback: number): number {
|
||||
const raw = process.env[envKey]?.trim();
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
@@ -29,8 +23,13 @@ function getRatioNumber(
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function normalizeSkillEnvSuffix(skill: string | undefined): string | undefined {
|
||||
const suffix = skill?.trim().toUpperCase().replace(/[^A-Z0-9]/gu, "_");
|
||||
function normalizeSkillEnvSuffix(
|
||||
skill: string | undefined,
|
||||
): string | undefined {
|
||||
const suffix = skill
|
||||
?.trim()
|
||||
.toUpperCase()
|
||||
.replace(/[^A-Z0-9]/gu, "_");
|
||||
return suffix || undefined;
|
||||
}
|
||||
|
||||
@@ -282,6 +281,10 @@ export function getSupervisorLlmConfidenceThreshold(): number {
|
||||
return getRatioNumber("AGENT_SUPERVISOR_LLM_CONFIDENCE_THRESHOLD", 0.6);
|
||||
}
|
||||
|
||||
export function getSupervisorLlmResponseEnabled(): boolean {
|
||||
return getBoolean("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getEvalCalibrationMinCorr(): number {
|
||||
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
||||
}
|
||||
@@ -334,7 +337,11 @@ export function getAgentSkillMdMode(): boolean {
|
||||
return getBoolean("AGENT_SKILL_MD_MODE", false);
|
||||
}
|
||||
|
||||
export function getAgentRouterMode(): "off" | "static" | "learned-v1" | "learned-v2" {
|
||||
export function getAgentRouterMode():
|
||||
| "off"
|
||||
| "static"
|
||||
| "learned-v1"
|
||||
| "learned-v2" {
|
||||
const raw = process.env["AGENT_ROUTER_MODE"]?.trim();
|
||||
if (
|
||||
raw === "off" ||
|
||||
@@ -456,7 +463,9 @@ export function getOpenRouterEnabled(): boolean {
|
||||
}
|
||||
|
||||
export function getOpenRouterApiUrl(): string {
|
||||
return process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1";
|
||||
return (
|
||||
process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1"
|
||||
);
|
||||
}
|
||||
|
||||
export function getTier0Model(): string {
|
||||
@@ -573,10 +582,7 @@ export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
|
||||
getNodeEnv() === "prod" ? 0.5 : 1.0,
|
||||
);
|
||||
case 2:
|
||||
return getRatioNumber(
|
||||
"SHADOW_EVAL_SAMPLING_RATE_TIER2",
|
||||
1.0,
|
||||
);
|
||||
return getRatioNumber("SHADOW_EVAL_SAMPLING_RATE_TIER2", 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -597,11 +603,7 @@ export function getAlertThresholdEvalScoreDropPct(skill?: string): number {
|
||||
}
|
||||
|
||||
export function getAlertThresholdLlmErrorRate(skill?: string): number {
|
||||
return getRatioNumberForSkill(
|
||||
"ALERT_THRESHOLD_LLM_ERROR_RATE",
|
||||
skill,
|
||||
0.1,
|
||||
);
|
||||
return getRatioNumberForSkill("ALERT_THRESHOLD_LLM_ERROR_RATE", skill, 0.1);
|
||||
}
|
||||
|
||||
export function getAlertThresholdDriftPct(skill?: string): number {
|
||||
|
||||
+101
-66
@@ -49,30 +49,37 @@ function parseSlice(value: string | undefined): SliceKey {
|
||||
};
|
||||
}
|
||||
|
||||
function printRunSummary(report: {
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
cases: number;
|
||||
passRate: number;
|
||||
ruleAverageScore: number;
|
||||
averageLatencyMs: number;
|
||||
llmAverageOverall: number | null;
|
||||
llmJudgedCases: number;
|
||||
};
|
||||
baselineDelta?: {
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
averageLatencyMs: number | null;
|
||||
} | undefined;
|
||||
}, reportPath: string): void {
|
||||
function printRunSummary(
|
||||
report: {
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
cases: number;
|
||||
passRate: number;
|
||||
ruleAverageScore: number;
|
||||
averageLatencyMs: number;
|
||||
llmAverageOverall: number | null;
|
||||
llmJudgedCases: number;
|
||||
};
|
||||
baselineDelta?:
|
||||
| {
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
averageLatencyMs: number | null;
|
||||
}
|
||||
| undefined;
|
||||
},
|
||||
reportPath: string,
|
||||
): void {
|
||||
const llmLine =
|
||||
report.summary.llmAverageOverall === null
|
||||
? "n/a"
|
||||
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
|
||||
|
||||
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
||||
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
|
||||
console.info(
|
||||
`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`,
|
||||
);
|
||||
console.info(
|
||||
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
|
||||
);
|
||||
@@ -229,12 +236,9 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
if (command === "run") {
|
||||
const { loadEvalCases, resolveEvalSetPath, runEval } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const { publishEvalReportIfPossible } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const { loadEvalCases, resolveEvalSetPath, runEval } =
|
||||
await import("./runner.js");
|
||||
const { publishEvalReportIfPossible } = await import("./runner.js");
|
||||
const setName = getFlagString(flags, "set") ?? "onboarding/cases";
|
||||
const resolvedSet = resolveEvalSetPath(setName);
|
||||
const skill = getFlagString(flags, "skill") ?? resolvedSet.skill;
|
||||
@@ -269,7 +273,8 @@ async function main(): Promise<void> {
|
||||
if (command === "baseline") {
|
||||
const { updateEvalBaseline } = await import("./runner.js");
|
||||
const reportPath = getFlagString(flags, "report") ?? positionals[0];
|
||||
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
|
||||
const { baseline, baselinePath, storedInRedis } =
|
||||
await updateEvalBaseline(reportPath);
|
||||
console.info(
|
||||
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
|
||||
);
|
||||
@@ -298,14 +303,15 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
if (command === "promote") {
|
||||
const { appendCandidatesToJsonl, selectPromotionCandidates } = await import(
|
||||
"./promote.js"
|
||||
);
|
||||
const { appendCandidatesToJsonl, selectPromotionCandidates } =
|
||||
await import("./promote.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const candidates = await selectPromotionCandidates(mongo.connection, {
|
||||
limit: parsePositiveInteger(getFlagString(flags, "limit")),
|
||||
minLowScore: parsePositiveNumber(getFlagString(flags, "min-low-score")),
|
||||
minLowScore: parsePositiveNumber(
|
||||
getFlagString(flags, "min-low-score"),
|
||||
),
|
||||
});
|
||||
|
||||
if (hasFlag(flags, "dry-run")) {
|
||||
@@ -336,7 +342,8 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
if (command === "review") {
|
||||
const { parseReviewReason, runReviewCli } = await import("./review/cli.js");
|
||||
const { parseReviewReason, runReviewCli } =
|
||||
await import("./review/cli.js");
|
||||
const garnet = await connectGarnetForEval();
|
||||
try {
|
||||
const result = await runReviewCli(garnet.garnetClient, {
|
||||
@@ -356,9 +363,8 @@ async function main(): Promise<void> {
|
||||
|
||||
if (command === "calibrate") {
|
||||
const { getEvalCalibrationMinCorr } = await import("../config.js");
|
||||
const { computeCalibrationReport } = await import(
|
||||
"./review/calibration.js"
|
||||
);
|
||||
const { computeCalibrationReport } =
|
||||
await import("./review/calibration.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
const garnet = await connectGarnetForEval();
|
||||
try {
|
||||
@@ -392,19 +398,13 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
if (command === "train-prm") {
|
||||
const {
|
||||
getEvalPrmMinLabels,
|
||||
getEvalPrmWeightsPath,
|
||||
} = await import("../config.js");
|
||||
const { getEvalPrmMinLabels, getEvalPrmWeightsPath } =
|
||||
await import("../config.js");
|
||||
const { getEvalArchiveCollection } = await import("./archive.js");
|
||||
const { queryArchivedReviewLabels } = await import(
|
||||
"./review/label-archive.js"
|
||||
);
|
||||
const {
|
||||
buildPrmTrainingSamples,
|
||||
savePrmModel,
|
||||
trainRidgePrmModel,
|
||||
} = await import("./judges/prm.js");
|
||||
const { queryArchivedReviewLabels } =
|
||||
await import("./review/label-archive.js");
|
||||
const { buildPrmTrainingSamples, savePrmModel, trainRidgePrmModel } =
|
||||
await import("./judges/prm.js");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const labels = await queryArchivedReviewLabels(mongo.connection, {
|
||||
@@ -460,16 +460,13 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
if (command === "train-router") {
|
||||
const {
|
||||
getAgentRouterWeightsPath,
|
||||
getEvalPrmMinLabels,
|
||||
} = await import("../config.js");
|
||||
const { getAgentRouterWeightsPath, getEvalPrmMinLabels } =
|
||||
await import("../config.js");
|
||||
const { getEvalArchiveCollection } = await import("./archive.js");
|
||||
const { buildRouterTrainingSamples } = await import("./router-training.js");
|
||||
const {
|
||||
saveLearnedRouterModel,
|
||||
trainLearnedRouterModel,
|
||||
} = await import("@skills/onboarding/runtime/router");
|
||||
const { buildRouterTrainingSamples } =
|
||||
await import("./router-training.js");
|
||||
const { saveLearnedRouterModel, trainLearnedRouterModel } =
|
||||
await import("@skills/onboarding/runtime/router");
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const archiveLimit =
|
||||
@@ -501,7 +498,9 @@ async function main(): Promise<void> {
|
||||
|
||||
const model = trainLearnedRouterModel(samples, {
|
||||
epochs: parsePositiveInteger(getFlagString(flags, "epochs")),
|
||||
learningRate: parsePositiveNumber(getFlagString(flags, "learning-rate")),
|
||||
learningRate: parsePositiveNumber(
|
||||
getFlagString(flags, "learning-rate"),
|
||||
),
|
||||
l2: parsePositiveNumber(getFlagString(flags, "l2")),
|
||||
});
|
||||
saveLearnedRouterModel(outputPath, model);
|
||||
@@ -514,6 +513,46 @@ async function main(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "router-shadow-report") {
|
||||
const { getAgentRouterWeightsPath } = await import("../config.js");
|
||||
const { loadLearnedRouterModel } =
|
||||
await import("@skills/onboarding/runtime/router");
|
||||
const { buildRouterShadowReport, formatRouterShadowReport } =
|
||||
await import("./router-shadow-report.js");
|
||||
const sinceMs = parseDurationMs(getFlagString(flags, "since"));
|
||||
const untilFlag = getFlagString(flags, "until");
|
||||
const untilMs = untilFlag ? parseDurationMs(untilFlag) : 0;
|
||||
const now = Date.now();
|
||||
const weightsPath =
|
||||
getFlagString(flags, "weights") ?? getAgentRouterWeightsPath();
|
||||
const reportOptions = {
|
||||
since: new Date(now - sinceMs),
|
||||
until: new Date(now - untilMs),
|
||||
weightsPath,
|
||||
};
|
||||
|
||||
if (!loadLearnedRouterModel(weightsPath)) {
|
||||
const report = await buildRouterShadowReport(
|
||||
{ readStreamRange: () => Promise.resolve([]) },
|
||||
reportOptions,
|
||||
);
|
||||
console.info(formatRouterShadowReport(report));
|
||||
return;
|
||||
}
|
||||
|
||||
const garnet = await connectGarnetForEval();
|
||||
try {
|
||||
const report = await buildRouterShadowReport(
|
||||
garnet.garnetClient,
|
||||
reportOptions,
|
||||
);
|
||||
console.info(formatRouterShadowReport(report));
|
||||
} finally {
|
||||
await garnet.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "propose") {
|
||||
const { createEvalLlmClient } = await import("./runner.js");
|
||||
const {
|
||||
@@ -559,23 +598,22 @@ async function main(): Promise<void> {
|
||||
|
||||
if (command === "tournament") {
|
||||
const { createEvalLlmClient } = await import("./runner.js");
|
||||
const {
|
||||
loadTournamentCases,
|
||||
runTournament,
|
||||
saveTournamentResult,
|
||||
} = await import("./loop/tournament.js");
|
||||
const { loadTournamentCases, runTournament, saveTournamentResult } =
|
||||
await import("./loop/tournament.js");
|
||||
const variantIds = parseCsv(
|
||||
getFlagString(flags, "variants") ?? positionals[0],
|
||||
);
|
||||
if (variantIds.length < 2) {
|
||||
throw new Error(
|
||||
'eval:tournament requires --variants=control,<variantId>',
|
||||
"eval:tournament requires --variants=control,<variantId>",
|
||||
);
|
||||
}
|
||||
|
||||
const mongo = await connectMongoForEval();
|
||||
try {
|
||||
const maxMatches = parsePositiveInteger(getFlagString(flags, "max-matches"));
|
||||
const maxMatches = parsePositiveInteger(
|
||||
getFlagString(flags, "max-matches"),
|
||||
);
|
||||
const skill = getFlagString(flags, "skill");
|
||||
const cases = await loadTournamentCases(mongo.connection, {
|
||||
caseIds: parseCsv(getFlagString(flags, "case-ids")),
|
||||
@@ -616,10 +654,7 @@ async function main(): Promise<void> {
|
||||
|
||||
throw new Error(`Unknown eval command: ${command}`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ err: error },
|
||||
`[eval] "${command}" failed`,
|
||||
);
|
||||
logger.error({ err: error }, `[eval] "${command}" failed`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
ROUTER_FEATURE_NAMES,
|
||||
ROUTING_DECISION_KINDS,
|
||||
type LearnedRouterModel,
|
||||
} from "@skills/onboarding/runtime/router";
|
||||
import {
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
} from "@skills/supervisor/review-queue";
|
||||
import { BACKGROUND_STREAM_KEY } from "./shadow-bus.js";
|
||||
import {
|
||||
buildRouterShadowReport,
|
||||
formatRouterShadowReport,
|
||||
} from "./router-shadow-report.js";
|
||||
|
||||
class FakeGarnet {
|
||||
calls = 0;
|
||||
|
||||
constructor(private readonly streams: Record<string, StreamEntry[]> = {}) {}
|
||||
|
||||
readStreamRange(key: string): Promise<StreamEntry[]> {
|
||||
this.calls += 1;
|
||||
return Promise.resolve(this.streams[key] ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
function writeModel(): string {
|
||||
const weights = ROUTING_DECISION_KINDS.map(() =>
|
||||
ROUTER_FEATURE_NAMES.map(() => 0),
|
||||
);
|
||||
const tier2Index = ROUTING_DECISION_KINDS.indexOf("tier2-full");
|
||||
const tier2Weights = weights[tier2Index];
|
||||
if (tier2Weights) {
|
||||
tier2Weights[0] = 10;
|
||||
}
|
||||
const model: LearnedRouterModel = {
|
||||
decisionKinds: [...ROUTING_DECISION_KINDS],
|
||||
featureNames: [...ROUTER_FEATURE_NAMES],
|
||||
trainedAt: "2026-05-18T00:00:00.000Z",
|
||||
version: 1,
|
||||
weights,
|
||||
};
|
||||
const path = join(
|
||||
mkdtempSync(join(tmpdir(), "router-shadow-")),
|
||||
"model.json",
|
||||
);
|
||||
writeFileSync(path, `${JSON.stringify(model)}\n`);
|
||||
return path;
|
||||
}
|
||||
|
||||
function streamEntry(id: string, message: Record<string, string>): StreamEntry {
|
||||
return { id, message };
|
||||
}
|
||||
|
||||
describe("buildRouterShadowReport", () => {
|
||||
it("reports missing weights without reading Garnet streams", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
|
||||
const report = await buildRouterShadowReport(garnet, {
|
||||
since: new Date("2026-05-18T00:00:00.000Z"),
|
||||
until: new Date("2026-05-18T01:00:00.000Z"),
|
||||
weightsPath: "/path/that/does/not/exist.json",
|
||||
});
|
||||
|
||||
expect(report).toMatchObject({
|
||||
agreementRate: 1,
|
||||
status: "weights-missing",
|
||||
totalSamples: 0,
|
||||
});
|
||||
expect(report.notes[0]).toContain("Router weights missing");
|
||||
expect(garnet.calls).toBe(0);
|
||||
});
|
||||
|
||||
it("compares static and learned router decisions from supervisor outcomes", async () => {
|
||||
const weightsPath = writeModel();
|
||||
const garnet = new FakeGarnet({
|
||||
[BACKGROUND_STREAM_KEY]: [
|
||||
streamEntry("10-0", {
|
||||
classified: JSON.stringify({
|
||||
confidence: 0.7,
|
||||
intent: { kind: "status-question" },
|
||||
source: "llm",
|
||||
}),
|
||||
createdAt: "2026-05-18T00:00:30.000Z",
|
||||
derived: JSON.stringify({
|
||||
allowShortResponse: false,
|
||||
intent: { kind: "status-question" },
|
||||
phase: { kind: "deadline-near", remaining: [] },
|
||||
responseMode: "llm",
|
||||
routingDecision: { kind: "tier1-direct", skipPlanner: false },
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 512,
|
||||
tier: 1,
|
||||
}),
|
||||
input: JSON.stringify({
|
||||
request: {
|
||||
chatroomId: "room-1",
|
||||
nickname: "tester",
|
||||
requestId: "request-1",
|
||||
skill: "onboarding",
|
||||
trigger: "user-message",
|
||||
userId: "user-1",
|
||||
userMessage: "아직 남은 절차가 있나요?",
|
||||
},
|
||||
}),
|
||||
output: JSON.stringify({
|
||||
confidence: 1,
|
||||
requestId: "request-1",
|
||||
source: "llm",
|
||||
text: "남은 절차를 확인해드릴게요.",
|
||||
}),
|
||||
skill: "onboarding",
|
||||
}),
|
||||
],
|
||||
[SUPERVISOR_RESOLUTIONS_STREAM_KEY]: [
|
||||
streamEntry("2-0", {
|
||||
action: "manual-reply",
|
||||
resolvedAt: "2026-05-18T00:02:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
reviewId: "1-0",
|
||||
type: "resolution",
|
||||
}),
|
||||
],
|
||||
[SUPERVISOR_REVIEW_STREAM_KEY]: [
|
||||
streamEntry("1-0", {
|
||||
chatroomId: "room-1",
|
||||
packet: JSON.stringify({
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "low-confidence",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
summary: "deadline status question",
|
||||
userGoal: "status",
|
||||
}),
|
||||
queuedAt: "2026-05-18T00:01:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
type: "supervisor-handoff",
|
||||
userId: "user-1",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const report = await buildRouterShadowReport(garnet, {
|
||||
since: new Date("2026-05-18T00:00:00.000Z"),
|
||||
until: new Date("2026-05-18T01:00:00.000Z"),
|
||||
weightsPath,
|
||||
});
|
||||
|
||||
expect(report).toMatchObject({
|
||||
agreementRate: 0,
|
||||
divergences: [
|
||||
{
|
||||
count: 1,
|
||||
examples: ["request-1"],
|
||||
learnedDecision: "tier2-full",
|
||||
staticDecision: "tier1-direct",
|
||||
},
|
||||
],
|
||||
skippedSamples: 0,
|
||||
status: "ok",
|
||||
totalSamples: 1,
|
||||
});
|
||||
expect(formatRouterShadowReport(report)).toContain(
|
||||
"tier1-direct\ttier2-full\t1\trequest-1",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,249 @@
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import {
|
||||
decideLearnedRouting,
|
||||
decidePreGenerationRouting,
|
||||
loadLearnedRouterModel,
|
||||
routerDecisionKind,
|
||||
type RoutingDecisionKind,
|
||||
type RouterInput,
|
||||
} from "@skills/onboarding/runtime/router";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
loadSupervisorOutcomes,
|
||||
type SupervisorOutcome,
|
||||
} from "./router-training-signals.js";
|
||||
import type { ShadowEvalEnvelope } from "./shadow-bus.js";
|
||||
|
||||
export interface RouterShadowDivergence {
|
||||
staticDecision: string;
|
||||
learnedDecision: string;
|
||||
count: number;
|
||||
examples: string[];
|
||||
}
|
||||
|
||||
export type RouterShadowReportStatus = "ok" | "weights-missing";
|
||||
|
||||
export interface RouterShadowReport {
|
||||
totalSamples: number;
|
||||
agreementRate: number;
|
||||
divergences: RouterShadowDivergence[];
|
||||
generatedAt: string;
|
||||
windowFrom: string;
|
||||
windowUntil: string;
|
||||
status: RouterShadowReportStatus;
|
||||
skippedSamples: number;
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export interface RouterShadowReportOptions {
|
||||
since: Date;
|
||||
until: Date;
|
||||
weightsPath?: string | undefined;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function isResponseMode(value: unknown): value is DerivedState["responseMode"] {
|
||||
return (
|
||||
value === "skip" ||
|
||||
value === "template" ||
|
||||
value === "llm" ||
|
||||
value === "llm-with-tools"
|
||||
);
|
||||
}
|
||||
|
||||
function isClassifiedIntent(value: unknown): value is ClassifiedIntent {
|
||||
const record = asRecord(value);
|
||||
const intent = asRecord(record["intent"]);
|
||||
return (
|
||||
typeof intent["kind"] === "string" &&
|
||||
typeof record["confidence"] === "number" &&
|
||||
Number.isFinite(record["confidence"]) &&
|
||||
(record["source"] === "regex" || record["source"] === "llm")
|
||||
);
|
||||
}
|
||||
|
||||
function isDerivedState(value: unknown): value is DerivedState {
|
||||
const record = asRecord(value);
|
||||
const phase = asRecord(record["phase"]);
|
||||
const intent = asRecord(record["intent"]);
|
||||
const tier = record["tier"];
|
||||
return (
|
||||
typeof phase["kind"] === "string" &&
|
||||
typeof intent["kind"] === "string" &&
|
||||
typeof record["shouldRespond"] === "boolean" &&
|
||||
isResponseMode(record["responseMode"]) &&
|
||||
typeof record["suggestedMaxTokens"] === "number" &&
|
||||
Number.isFinite(record["suggestedMaxTokens"]) &&
|
||||
typeof record["allowShortResponse"] === "boolean" &&
|
||||
(tier === 0 || tier === 1 || tier === 2)
|
||||
);
|
||||
}
|
||||
|
||||
function isOnboardingShadowEntry(
|
||||
shadow: ShadowEvalEnvelope,
|
||||
): shadow is ShadowEvalEnvelope & {
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState;
|
||||
request: AgentRequest;
|
||||
} {
|
||||
return (
|
||||
(shadow.skill === "onboarding" || shadow.request.skill === "onboarding") &&
|
||||
isClassifiedIntent(shadow.classified) &&
|
||||
isDerivedState(shadow.derived)
|
||||
);
|
||||
}
|
||||
|
||||
function routerInputFromOutcome(
|
||||
outcome: SupervisorOutcome,
|
||||
): RouterInput | null {
|
||||
const shadow = outcome.shadowEvalEntry;
|
||||
if (!shadow || !isOnboardingShadowEntry(shadow)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
classified: shadow.classified,
|
||||
derived: shadow.derived,
|
||||
request: shadow.request,
|
||||
session: null,
|
||||
};
|
||||
}
|
||||
|
||||
function addDivergence(
|
||||
groups: Map<string, RouterShadowDivergence>,
|
||||
staticDecision: RoutingDecisionKind,
|
||||
learnedDecision: RoutingDecisionKind,
|
||||
requestId: string,
|
||||
): void {
|
||||
const key = `${staticDecision}->${learnedDecision}`;
|
||||
const current = groups.get(key) ?? {
|
||||
count: 0,
|
||||
examples: [],
|
||||
learnedDecision,
|
||||
staticDecision,
|
||||
};
|
||||
current.count += 1;
|
||||
if (current.examples.length < 5) {
|
||||
current.examples.push(requestId);
|
||||
}
|
||||
groups.set(key, current);
|
||||
}
|
||||
|
||||
function roundRate(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
export async function buildRouterShadowReport(
|
||||
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||
options: RouterShadowReportOptions,
|
||||
): Promise<RouterShadowReport> {
|
||||
const generatedAt = new Date().toISOString();
|
||||
const baseReport = {
|
||||
generatedAt,
|
||||
windowFrom: options.since.toISOString(),
|
||||
windowUntil: options.until.toISOString(),
|
||||
};
|
||||
const model = loadLearnedRouterModel(options.weightsPath);
|
||||
if (!model) {
|
||||
return {
|
||||
...baseReport,
|
||||
agreementRate: 1,
|
||||
divergences: [],
|
||||
notes: [
|
||||
"Router weights missing or invalid; set --weights or AGENT_ROUTER_WEIGHTS_PATH.",
|
||||
],
|
||||
skippedSamples: 0,
|
||||
status: "weights-missing",
|
||||
totalSamples: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const outcomes = await loadSupervisorOutcomes(
|
||||
garnetClient,
|
||||
options.since,
|
||||
options.until,
|
||||
);
|
||||
const divergences = new Map<string, RouterShadowDivergence>();
|
||||
let totalSamples = 0;
|
||||
let matchingSamples = 0;
|
||||
let skippedSamples = 0;
|
||||
|
||||
for (const outcome of outcomes) {
|
||||
const input = routerInputFromOutcome(outcome);
|
||||
if (!input) {
|
||||
skippedSamples += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const staticDecision = routerDecisionKind(
|
||||
decidePreGenerationRouting(input),
|
||||
);
|
||||
const learnedDecision = routerDecisionKind(
|
||||
decideLearnedRouting(input, model),
|
||||
);
|
||||
totalSamples += 1;
|
||||
if (staticDecision === learnedDecision) {
|
||||
matchingSamples += 1;
|
||||
continue;
|
||||
}
|
||||
addDivergence(
|
||||
divergences,
|
||||
staticDecision,
|
||||
learnedDecision,
|
||||
input.request.requestId,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
...baseReport,
|
||||
agreementRate:
|
||||
totalSamples === 0 ? 1 : roundRate(matchingSamples / totalSamples),
|
||||
divergences: [...divergences.values()].sort(
|
||||
(left, right) => right.count - left.count,
|
||||
),
|
||||
notes: [],
|
||||
skippedSamples,
|
||||
status: "ok",
|
||||
totalSamples,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatRouterShadowReport(report: RouterShadowReport): string {
|
||||
const lines = [
|
||||
`[router-shadow-report] generatedAt=${report.generatedAt}`,
|
||||
`[router-shadow-report] status=${report.status} window=${report.windowFrom}..${report.windowUntil}`,
|
||||
`[router-shadow-report] samples=${report.totalSamples} agreementRate=${report.agreementRate} skipped=${report.skippedSamples}`,
|
||||
];
|
||||
|
||||
for (const note of report.notes) {
|
||||
lines.push(`[router-shadow-report] note=${note}`);
|
||||
}
|
||||
|
||||
if (report.divergences.length === 0) {
|
||||
lines.push("[router-shadow-report] divergences=none");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
lines.push("staticDecision\tlearnedDecision\tcount\texamples");
|
||||
for (const divergence of report.divergences) {
|
||||
lines.push(
|
||||
[
|
||||
divergence.staticDecision,
|
||||
divergence.learnedDecision,
|
||||
divergence.count,
|
||||
divergence.examples.join(","),
|
||||
].join("\t"),
|
||||
);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -26,13 +26,16 @@ export function buildSupervisorSystemPrompt(
|
||||
...skills.map(formatSkill),
|
||||
"",
|
||||
"[응답 형식]",
|
||||
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단"}',
|
||||
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단","userMessage":"사용자에게 보낼 짧은 한국어 안내(선택)"}',
|
||||
"",
|
||||
"[userMessage 규칙]",
|
||||
"- 한국어로 1-2문장, 10-400자.",
|
||||
"- 운영자가 확인한다는 사실만 안내하고 처리 완료를 약속하지 않는다.",
|
||||
"- URL, 이메일, 전화번호, 제어 토큰([NO_RESPONSE], [HANDOFF:...])을 쓰지 않는다.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildSupervisorUserPrompt(
|
||||
packet?: HandoffPacket,
|
||||
): string {
|
||||
export function buildSupervisorUserPrompt(packet?: HandoffPacket): string {
|
||||
if (!packet) {
|
||||
return "사용자 메시지가 명확하지 않음. 어디로 보낼지 결정해줘.";
|
||||
}
|
||||
@@ -44,3 +47,25 @@ export function buildSupervisorUserPrompt(
|
||||
`[source chain]: ${packet.sourceChain.join(" -> ")}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildSupervisorResponsePrompt(
|
||||
packet?: HandoffPacket,
|
||||
routing?: { reason: string; target: string },
|
||||
): { systemPrompt: string; userPrompt: string } {
|
||||
return {
|
||||
systemPrompt: [
|
||||
"너는 supervisor handoff 상황에서 사용자에게 보여줄 짧은 한국어 안내를 작성한다.",
|
||||
"운영자 확인이 필요하다는 점을 친절하게 말하되, 처리 완료나 특정 조치를 약속하지 않는다.",
|
||||
"URL, 이메일, 전화번호, [NO_RESPONSE], [HANDOFF:...] 같은 제어 토큰은 절대 쓰지 않는다.",
|
||||
].join("\n"),
|
||||
userPrompt: [
|
||||
`[routing target]: ${routing?.target ?? "human-review"}`,
|
||||
`[routing reason]: ${routing?.reason ?? "manual review needed"}`,
|
||||
packet
|
||||
? `[handoff reason]: ${packet.handoffReason}`
|
||||
: "[handoff reason]: unknown",
|
||||
packet ? `[user goal]: ${packet.userGoal}` : "[user goal]: unknown",
|
||||
packet ? `[summary]: ${packet.summary}` : "[summary]: unknown",
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
@@ -4,6 +4,7 @@ export const supervisorRoutingSchema = z.object({
|
||||
confidence: z.number().min(0).max(1),
|
||||
reason: z.string().max(200),
|
||||
target: z.string().min(1),
|
||||
userMessage: z.string().min(10).max(400).optional(),
|
||||
});
|
||||
|
||||
export type SupervisorRoutingDecision = z.infer<typeof supervisorRoutingSchema>;
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||
@@ -8,7 +8,8 @@ import type { SkillRegistry } from "@skills/skill-loader";
|
||||
import { SupervisorSkill } from "./skill.js";
|
||||
|
||||
class FakeGarnet {
|
||||
readonly streams: Array<{ key: string; fields: Record<string, unknown> }> = [];
|
||||
readonly streams: Array<{ key: string; fields: Record<string, unknown> }> =
|
||||
[];
|
||||
|
||||
appendStream(key: string, fields: Record<string, unknown>): Promise<string> {
|
||||
this.streams.push({ key, fields });
|
||||
@@ -30,6 +31,14 @@ class FakeNats {
|
||||
}
|
||||
|
||||
class FakeLlm {
|
||||
constructor(
|
||||
private readonly content: Record<string, unknown> = {
|
||||
confidence: 0.9,
|
||||
reason: "광고 규칙 문의라 moderation이 적합",
|
||||
target: "moderation",
|
||||
},
|
||||
) {}
|
||||
|
||||
chatCompletion(): Promise<{
|
||||
content: string;
|
||||
endpoint: string;
|
||||
@@ -38,11 +47,7 @@ class FakeLlm {
|
||||
raw: object;
|
||||
}> {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
confidence: 0.9,
|
||||
reason: "광고 규칙 문의라 moderation이 적합",
|
||||
target: "moderation",
|
||||
}),
|
||||
content: JSON.stringify(this.content),
|
||||
endpoint: "memory",
|
||||
message: { content: "", role: "assistant" },
|
||||
provider: "llama",
|
||||
@@ -84,6 +89,10 @@ const sourceResponse: AgentResponse = {
|
||||
};
|
||||
|
||||
describe("SupervisorSkill", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("queues review and publishes alert for handoffs", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
@@ -111,6 +120,7 @@ describe("SupervisorSkill", () => {
|
||||
expect(response.text).toContain("운영자");
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "rules",
|
||||
source: "template",
|
||||
supervisorReason: "max-hop",
|
||||
suggestedTarget: "onboarding",
|
||||
});
|
||||
@@ -169,6 +179,7 @@ describe("SupervisorSkill", () => {
|
||||
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "llm",
|
||||
source: "template",
|
||||
supervisorReason: "광고 규칙 문의라 moderation이 적합",
|
||||
suggestedTarget: "moderation",
|
||||
});
|
||||
@@ -177,4 +188,57 @@ describe("SupervisorSkill", () => {
|
||||
target: "moderation",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses safe LLM userMessage for supervisor replies", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const skill = new SupervisorSkill(
|
||||
garnet,
|
||||
nats,
|
||||
new FakeLlm({
|
||||
confidence: 0.9,
|
||||
reason: "광고 규칙 문의라 moderation이 적합",
|
||||
target: "moderation",
|
||||
userMessage: "이 내용은 운영자가 확인해야 해서 담당자에게 전달했어요.",
|
||||
}) as unknown as Pick<LlmClient, "chatCompletion">,
|
||||
registry,
|
||||
);
|
||||
|
||||
const response = await skill.handle(baseRequest, []);
|
||||
|
||||
expect(response.text).toBe(
|
||||
"이 내용은 운영자가 확인해야 해서 담당자에게 전달했어요.",
|
||||
);
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "llm",
|
||||
source: "llm",
|
||||
suggestedTarget: "moderation",
|
||||
});
|
||||
expect(response.source).toBe("template");
|
||||
});
|
||||
|
||||
it("falls back to template when LLM userMessage is unsafe or disabled", async () => {
|
||||
vi.stubEnv("AGENT_SUPERVISOR_LLM_RESPONSE_ENABLED", "false");
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const skill = new SupervisorSkill(
|
||||
garnet,
|
||||
nats,
|
||||
new FakeLlm({
|
||||
confidence: 0.9,
|
||||
reason: "광고 규칙 문의라 moderation이 적합",
|
||||
target: "moderation",
|
||||
userMessage: "자세한 내용은 https://example.com 에서 확인해 주세요.",
|
||||
}) as unknown as Pick<LlmClient, "chatCompletion">,
|
||||
registry,
|
||||
);
|
||||
|
||||
const response = await skill.handle(baseRequest, []);
|
||||
|
||||
expect(response.text).toContain("운영자");
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "llm",
|
||||
source: "template",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import { getLogger } from "@obs/logger";
|
||||
import {
|
||||
getSupervisorLlmConfidenceThreshold,
|
||||
getSupervisorLlmEnabled,
|
||||
getSupervisorLlmResponseEnabled,
|
||||
} from "../../config.js";
|
||||
import {
|
||||
buildSupervisorSystemPrompt,
|
||||
@@ -42,13 +43,47 @@ const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
riskTier: "medium",
|
||||
};
|
||||
|
||||
const TEMPLATE_HUMAN_REVIEW_REPLY =
|
||||
"확인이 필요한 내용이라 운영자에게 전달했어요.";
|
||||
|
||||
const UNSAFE_USER_MESSAGE_PATTERNS = [
|
||||
/https?:\/\//iu,
|
||||
/www\./iu,
|
||||
/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/iu,
|
||||
/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/u,
|
||||
/\[(?:NO_RESPONSE|HANDOFF(?::[^\]]*)?)\]/iu,
|
||||
] as const;
|
||||
|
||||
function getSafeSupervisorUserMessage(
|
||||
decision: SupervisorRoutingDecision | null,
|
||||
): {
|
||||
source: "llm" | "template";
|
||||
text: string;
|
||||
} {
|
||||
const message = decision?.userMessage?.trim();
|
||||
if (
|
||||
!getSupervisorLlmResponseEnabled() ||
|
||||
!message ||
|
||||
message.length < 10 ||
|
||||
message.length > 400 ||
|
||||
UNSAFE_USER_MESSAGE_PATTERNS.some((pattern) => pattern.test(message))
|
||||
) {
|
||||
return { source: "template", text: TEMPLATE_HUMAN_REVIEW_REPLY };
|
||||
}
|
||||
|
||||
return { source: "llm", text: message };
|
||||
}
|
||||
|
||||
export class SupervisorSkill implements Skill {
|
||||
readonly name = "supervisor";
|
||||
readonly priority = 10;
|
||||
private readonly alertService: AlertService;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||
private readonly garnetClient: Pick<
|
||||
GarnetClient,
|
||||
"appendStream" | "setStringIfAbsent"
|
||||
>,
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||
@@ -77,7 +112,9 @@ export class SupervisorSkill implements Skill {
|
||||
const useLlmRoute =
|
||||
llmDecision !== null &&
|
||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
|
||||
const target = useLlmRoute
|
||||
? llmDecision.target
|
||||
: routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
@@ -107,16 +144,21 @@ export class SupervisorSkill implements Skill {
|
||||
},
|
||||
);
|
||||
|
||||
const userMessage = getSafeSupervisorUserMessage(
|
||||
useLlmRoute ? llmDecision : null,
|
||||
);
|
||||
|
||||
return {
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
routedBy: useLlmRoute ? "llm" : "rules",
|
||||
source: userMessage.source,
|
||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
text: userMessage.text,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -167,6 +209,8 @@ export class SupervisorSkill implements Skill {
|
||||
}
|
||||
|
||||
private isKnownTarget(target: string): boolean {
|
||||
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
|
||||
return (
|
||||
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user