Close multi-agent plan and add router shadow reporting

This commit is contained in:
dahyeon committed 2026-05-18 19:34:11 -07:00
1 parent f1b30d716d
commit 8156e8c07e
15 files changed
+1303 -204

No files matched your search

+48 -4
View File
@@ -17,6 +17,7 @@ import { getLogger } from "@obs/logger";
import {
getSupervisorLlmConfidenceThreshold,
getSupervisorLlmEnabled,
getSupervisorLlmResponseEnabled,
} from "../../config.js";
import {
buildSupervisorSystemPrompt,
@@ -42,13 +43,47 @@ const CAPABILITY_CARD: SkillCapabilityCard = {
riskTier: "medium",
};
const TEMPLATE_HUMAN_REVIEW_REPLY =
"확인이 필요한 내용이라 운영자에게 전달했어요.";
const UNSAFE_USER_MESSAGE_PATTERNS = [
/https?:\/\//iu,
/www\./iu,
/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/iu,
/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/u,
/\[(?:NO_RESPONSE|HANDOFF(?::[^\]]*)?)\]/iu,
] as const;
function getSafeSupervisorUserMessage(
decision: SupervisorRoutingDecision | null,
): {
source: "llm" | "template";
text: string;
} {
const message = decision?.userMessage?.trim();
if (
!getSupervisorLlmResponseEnabled() ||
!message ||
message.length < 10 ||
message.length > 400 ||
UNSAFE_USER_MESSAGE_PATTERNS.some((pattern) => pattern.test(message))
) {
return { source: "template", text: TEMPLATE_HUMAN_REVIEW_REPLY };
}
return { source: "llm", text: message };
}
export class SupervisorSkill implements Skill {
readonly name = "supervisor";
readonly priority = 10;
private readonly alertService: AlertService;
constructor(
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
private readonly garnetClient: Pick<
GarnetClient,
"appendStream" | "setStringIfAbsent"
>,
natsClient: Pick<NatsClient, "publish">,
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
private readonly skillRegistry?: SkillRegistry | undefined,
@@ -77,7 +112,9 @@ export class SupervisorSkill implements Skill {
const useLlmRoute =
llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
const target = useLlmRoute
? llmDecision.target
: routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope";
await appendSupervisorReview(this.garnetClient, {
@@ -107,16 +144,21 @@ export class SupervisorSkill implements Skill {
},
);
const userMessage = getSafeSupervisorUserMessage(
useLlmRoute ? llmDecision : null,
);
return {
confidence: 0.6,
metadata: {
routedBy: useLlmRoute ? "llm" : "rules",
source: userMessage.source,
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
suggestedTarget: target,
},
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
text: userMessage.text,
};
}
@@ -167,6 +209,8 @@ export class SupervisorSkill implements Skill {
}
private isKnownTarget(target: string): boolean {
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
return (
target === "human-review" || Boolean(this.skillRegistry?.get(target))
);
}
}