256 lines
7.2 KiB
TypeScript
256 lines
7.2 KiB
TypeScript
import type {
|
|
AgentRequest,
|
|
AgentResponse,
|
|
ConversationMessage,
|
|
} from "@agent/types";
|
|
import { readHandoffPacket } from "@agent/handoff-packet";
|
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
|
import type { GarnetClient } from "@common/garnet";
|
|
import type { NatsClient } from "@common/nats";
|
|
import type { LlmClient } from "@llm/llm-client";
|
|
import type {
|
|
EscalationHistoryService,
|
|
UserEscalationHistory,
|
|
} from "../../memory/escalation-history.js";
|
|
import { AlertService } from "@obs/alert";
|
|
import { metrics } from "@obs/metrics";
|
|
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
|
|
import type { Skill } from "@skills/skill";
|
|
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
|
import { getLogger } from "@obs/logger";
|
|
import {
|
|
getSupervisorLlmConfidenceThreshold,
|
|
getSupervisorLlmEnabled,
|
|
getSupervisorLlmResponseEnabled,
|
|
} from "../../config.js";
|
|
import {
|
|
buildSupervisorSystemPrompt,
|
|
buildSupervisorUserPrompt,
|
|
} from "./capability-prompt.js";
|
|
import { appendSupervisorReview } from "./review-queue.js";
|
|
import { routeSupervisorTarget } from "./routing-rules.js";
|
|
import {
|
|
supervisorRoutingSchema,
|
|
type SupervisorRoutingDecision,
|
|
} from "./schema.js";
|
|
|
|
const CAPABILITY_CARD: SkillCapabilityCard = {
|
|
canHandle: ["*"],
|
|
cannotHandle: [],
|
|
domain: "meta",
|
|
handoffTargets: ["onboarding", "moderation"],
|
|
outputContract: {
|
|
format: "text",
|
|
maxChars: "500",
|
|
requiredFields: "text",
|
|
},
|
|
riskTier: "medium",
|
|
};
|
|
|
|
const TEMPLATE_HUMAN_REVIEW_REPLY =
|
|
"확인이 필요한 내용이라 운영자에게 전달했어요.";
|
|
|
|
const UNSAFE_USER_MESSAGE_PATTERNS = [
|
|
/https?:\/\//iu,
|
|
/www\./iu,
|
|
/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/iu,
|
|
/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/u,
|
|
/\[(?:NO_RESPONSE|HANDOFF(?::[^\]]*)?)\]/iu,
|
|
] as const;
|
|
|
|
function getSafeSupervisorUserMessage(
|
|
decision: SupervisorRoutingDecision | null,
|
|
): {
|
|
source: "llm" | "template";
|
|
text: string;
|
|
} {
|
|
const message = decision?.userMessage?.trim();
|
|
if (
|
|
!getSupervisorLlmResponseEnabled() ||
|
|
!message ||
|
|
message.length < 10 ||
|
|
message.length > 400 ||
|
|
UNSAFE_USER_MESSAGE_PATTERNS.some((pattern) => pattern.test(message))
|
|
) {
|
|
return { source: "template", text: TEMPLATE_HUMAN_REVIEW_REPLY };
|
|
}
|
|
|
|
return { source: "llm", text: message };
|
|
}
|
|
|
|
export class SupervisorSkill implements Skill {
|
|
readonly name = "supervisor";
|
|
readonly priority = 10;
|
|
private readonly alertService: AlertService;
|
|
|
|
constructor(
|
|
private readonly garnetClient: Pick<
|
|
GarnetClient,
|
|
"appendStream" | "setStringIfAbsent"
|
|
>,
|
|
natsClient: Pick<NatsClient, "publish">,
|
|
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
|
private readonly skillRegistry?: SkillRegistry | undefined,
|
|
private readonly escalationHistoryService?: EscalationHistoryService | undefined,
|
|
) {
|
|
this.alertService = new AlertService(
|
|
garnetClient,
|
|
natsClient,
|
|
createSlackNotifierFromEnv(),
|
|
);
|
|
}
|
|
|
|
canHandle(request: AgentRequest): boolean {
|
|
return request.skill === this.name;
|
|
}
|
|
|
|
capabilityCard(): SkillCapabilityCard {
|
|
return CAPABILITY_CARD;
|
|
}
|
|
|
|
async handle(
|
|
request: AgentRequest,
|
|
_conversationHistory: ConversationMessage[],
|
|
): Promise<AgentResponse> {
|
|
const packet = readHandoffPacket(request);
|
|
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
|
const llmDecision = await this.routeByLlm(packet, escalationHistory);
|
|
const useLlmRoute =
|
|
llmDecision !== null &&
|
|
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
|
const target = useLlmRoute
|
|
? llmDecision.target
|
|
: routeSupervisorTarget(packet);
|
|
const reason = packet?.handoffReason ?? "out-of-scope";
|
|
|
|
const queuedAt = new Date().toISOString();
|
|
const reviewId = await appendSupervisorReview(this.garnetClient, {
|
|
chatroomId: request.chatroomId,
|
|
packet: JSON.stringify(packet ?? null),
|
|
queuedAt,
|
|
reason,
|
|
requestId: request.requestId,
|
|
target: target ?? "human-review",
|
|
userId: request.userId,
|
|
});
|
|
await this.escalationHistoryService?.recordEscalation({
|
|
chatroomId: request.chatroomId,
|
|
packetSummary: packet?.userGoal ?? packet?.summary ?? "",
|
|
queuedAt,
|
|
reason,
|
|
reviewId,
|
|
sourceChain: packet?.sourceChain ?? [],
|
|
target: target ?? "human-review",
|
|
userId: request.userId,
|
|
});
|
|
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
|
|
|
|
await this.alertService.sendAlert(
|
|
{
|
|
kind: "supervisor-handoff",
|
|
metric: "agent.supervisor_handoff",
|
|
severity: "warn",
|
|
skill: this.name,
|
|
suggestedAction: "Review the supervisor handoff queue.",
|
|
threshold: 0,
|
|
timestamp: new Date().toISOString(),
|
|
value: 1,
|
|
window: "request",
|
|
},
|
|
{
|
|
dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`,
|
|
},
|
|
);
|
|
|
|
const userMessage = getSafeSupervisorUserMessage(
|
|
useLlmRoute ? llmDecision : null,
|
|
);
|
|
|
|
return {
|
|
confidence: 0.6,
|
|
metadata: {
|
|
routedBy: useLlmRoute ? "llm" : "rules",
|
|
source: userMessage.source,
|
|
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
|
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
|
|
suggestedTarget: target,
|
|
},
|
|
requestId: request.requestId,
|
|
source: "template",
|
|
text: userMessage.text,
|
|
};
|
|
}
|
|
|
|
private async routeByLlm(
|
|
packet?: HandoffPacket,
|
|
escalationHistory?: UserEscalationHistory | null,
|
|
): Promise<SupervisorRoutingDecision | null> {
|
|
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const result = await this.llmClient.chatCompletion(
|
|
[
|
|
{
|
|
content: buildSupervisorSystemPrompt(
|
|
this.skillRegistry,
|
|
packet,
|
|
escalationHistory,
|
|
),
|
|
role: "system",
|
|
},
|
|
{
|
|
content: buildSupervisorUserPrompt(packet),
|
|
role: "user",
|
|
},
|
|
],
|
|
{
|
|
jsonMode: true,
|
|
maxTokens: 150,
|
|
temperature: 0,
|
|
tier: 0,
|
|
},
|
|
);
|
|
if (!result?.content.trim()) {
|
|
return null;
|
|
}
|
|
|
|
const parsed = supervisorRoutingSchema.safeParse(
|
|
JSON.parse(result.content) as unknown,
|
|
);
|
|
if (!parsed.success || !this.isKnownTarget(parsed.data.target)) {
|
|
return null;
|
|
}
|
|
return parsed.data;
|
|
} catch (error) {
|
|
getLogger().warn(
|
|
{ err: error },
|
|
"[Supervisor] LLM routing failed, falling back to rules",
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private isKnownTarget(target: string): boolean {
|
|
return (
|
|
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
|
);
|
|
}
|
|
|
|
private async loadEscalationHistory(userId: string) {
|
|
if (!this.escalationHistoryService) {
|
|
return null;
|
|
}
|
|
try {
|
|
return await this.escalationHistoryService.load(userId);
|
|
} catch (error) {
|
|
getLogger().warn(
|
|
{ err: error },
|
|
"[Supervisor] Failed to load escalation history",
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
}
|