Close multi-agent plan and add router shadow reporting
This commit is contained in:
1 parent
f1b30d716d
commit
8156e8c07e
15 files changed
+1303
-204
No files matched your search
@@ -17,6 +17,7 @@ import { getLogger } from "@obs/logger";
|
||||
import {
|
||||
getSupervisorLlmConfidenceThreshold,
|
||||
getSupervisorLlmEnabled,
|
||||
getSupervisorLlmResponseEnabled,
|
||||
} from "../../config.js";
|
||||
import {
|
||||
buildSupervisorSystemPrompt,
|
||||
@@ -42,13 +43,47 @@ const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
riskTier: "medium",
|
||||
};
|
||||
|
||||
const TEMPLATE_HUMAN_REVIEW_REPLY =
|
||||
"확인이 필요한 내용이라 운영자에게 전달했어요.";
|
||||
|
||||
const UNSAFE_USER_MESSAGE_PATTERNS = [
|
||||
/https?:\/\//iu,
|
||||
/www\./iu,
|
||||
/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/iu,
|
||||
/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/u,
|
||||
/\[(?:NO_RESPONSE|HANDOFF(?::[^\]]*)?)\]/iu,
|
||||
] as const;
|
||||
|
||||
function getSafeSupervisorUserMessage(
|
||||
decision: SupervisorRoutingDecision | null,
|
||||
): {
|
||||
source: "llm" | "template";
|
||||
text: string;
|
||||
} {
|
||||
const message = decision?.userMessage?.trim();
|
||||
if (
|
||||
!getSupervisorLlmResponseEnabled() ||
|
||||
!message ||
|
||||
message.length < 10 ||
|
||||
message.length > 400 ||
|
||||
UNSAFE_USER_MESSAGE_PATTERNS.some((pattern) => pattern.test(message))
|
||||
) {
|
||||
return { source: "template", text: TEMPLATE_HUMAN_REVIEW_REPLY };
|
||||
}
|
||||
|
||||
return { source: "llm", text: message };
|
||||
}
|
||||
|
||||
export class SupervisorSkill implements Skill {
|
||||
readonly name = "supervisor";
|
||||
readonly priority = 10;
|
||||
private readonly alertService: AlertService;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||
private readonly garnetClient: Pick<
|
||||
GarnetClient,
|
||||
"appendStream" | "setStringIfAbsent"
|
||||
>,
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||
@@ -77,7 +112,9 @@ export class SupervisorSkill implements Skill {
|
||||
const useLlmRoute =
|
||||
llmDecision !== null &&
|
||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
|
||||
const target = useLlmRoute
|
||||
? llmDecision.target
|
||||
: routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
@@ -107,16 +144,21 @@ export class SupervisorSkill implements Skill {
|
||||
},
|
||||
);
|
||||
|
||||
const userMessage = getSafeSupervisorUserMessage(
|
||||
useLlmRoute ? llmDecision : null,
|
||||
);
|
||||
|
||||
return {
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
routedBy: useLlmRoute ? "llm" : "rules",
|
||||
source: userMessage.source,
|
||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
text: userMessage.text,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -167,6 +209,8 @@ export class SupervisorSkill implements Skill {
|
||||
}
|
||||
|
||||
private isKnownTarget(target: string): boolean {
|
||||
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
|
||||
return (
|
||||
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user