import type { AgentRequest, AgentResponse, ConversationMessage, } from "@agent/types"; import { readHandoffPacket } from "@agent/handoff-packet"; import type { HandoffPacket } from "@agent/handoff-packet"; import type { GarnetClient } from "@common/garnet"; import type { NatsClient } from "@common/nats"; import type { LlmClient } from "@llm/llm-client"; import type { EscalationHistoryService, UserEscalationHistory, } from "../../memory/escalation-history.js"; import { AlertService } from "@obs/alert"; import { metrics } from "@obs/metrics"; import { createSlackNotifierFromEnv } from "@obs/slack-notifier"; import type { Skill } from "@skills/skill"; import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader"; import { getLogger } from "@obs/logger"; import { getSupervisorLlmConfidenceThreshold, getSupervisorLlmEnabled, getSupervisorLlmResponseEnabled, } from "../../config.js"; import { buildSupervisorSystemPrompt, buildSupervisorUserPrompt, } from "./capability-prompt.js"; import { appendSupervisorReview } from "./review-queue.js"; import { routeSupervisorTarget } from "./routing-rules.js"; import { supervisorRoutingSchema, type SupervisorRoutingDecision, } from "./schema.js"; const CAPABILITY_CARD: SkillCapabilityCard = { canHandle: ["*"], cannotHandle: [], domain: "meta", handoffTargets: ["onboarding", "moderation"], outputContract: { format: "text", maxChars: "500", requiredFields: "text", }, riskTier: "medium", }; const TEMPLATE_HUMAN_REVIEW_REPLY = "확인이 필요한 내용이라 운영자에게 전달했어요."; const UNSAFE_USER_MESSAGE_PATTERNS = [ /https?:\/\//iu, /www\./iu, /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/iu, /\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/u, /\[(?:NO_RESPONSE|HANDOFF(?::[^\]]*)?)\]/iu, ] as const; function getSafeSupervisorUserMessage( decision: SupervisorRoutingDecision | null, ): { source: "llm" | "template"; text: string; } { const message = decision?.userMessage?.trim(); if ( !getSupervisorLlmResponseEnabled() || !message || message.length < 10 || message.length > 400 || UNSAFE_USER_MESSAGE_PATTERNS.some((pattern) => pattern.test(message)) ) { return { source: "template", text: TEMPLATE_HUMAN_REVIEW_REPLY }; } return { source: "llm", text: message }; } export class SupervisorSkill implements Skill { readonly name = "supervisor"; readonly priority = 10; private readonly alertService: AlertService; constructor( private readonly garnetClient: Pick< GarnetClient, "appendStream" | "setStringIfAbsent" >, natsClient: Pick, private readonly llmClient?: Pick | undefined, private readonly skillRegistry?: SkillRegistry | undefined, private readonly escalationHistoryService?: EscalationHistoryService | undefined, ) { this.alertService = new AlertService( garnetClient, natsClient, createSlackNotifierFromEnv(), ); } canHandle(request: AgentRequest): boolean { return request.skill === this.name; } capabilityCard(): SkillCapabilityCard { return CAPABILITY_CARD; } async handle( request: AgentRequest, _conversationHistory: ConversationMessage[], ): Promise { const packet = readHandoffPacket(request); const escalationHistory = await this.loadEscalationHistory(request.userId); const llmDecision = await this.routeByLlm(packet, escalationHistory); const useLlmRoute = llmDecision !== null && llmDecision.confidence >= getSupervisorLlmConfidenceThreshold(); const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet); const reason = packet?.handoffReason ?? "out-of-scope"; const queuedAt = new Date().toISOString(); const reviewId = await appendSupervisorReview(this.garnetClient, { chatroomId: request.chatroomId, packet: JSON.stringify(packet ?? null), queuedAt, reason, requestId: request.requestId, target: target ?? "human-review", userId: request.userId, }); await this.escalationHistoryService?.recordEscalation({ chatroomId: request.chatroomId, packetSummary: packet?.userGoal ?? packet?.summary ?? "", queuedAt, reason, reviewId, sourceChain: packet?.sourceChain ?? [], target: target ?? "human-review", userId: request.userId, }); metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name }); await this.alertService.sendAlert( { kind: "supervisor-handoff", metric: "agent.supervisor_handoff", severity: "warn", skill: this.name, suggestedAction: "Review the supervisor handoff queue.", threshold: 0, timestamp: new Date().toISOString(), value: 1, window: "request", }, { dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`, }, ); const userMessage = getSafeSupervisorUserMessage( useLlmRoute ? llmDecision : null, ); return { confidence: 0.6, metadata: { routedBy: useLlmRoute ? "llm" : "rules", source: userMessage.source, supervisorReason: useLlmRoute ? llmDecision.reason : reason, supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0, suggestedTarget: target, }, requestId: request.requestId, source: "template", text: userMessage.text, }; } private async routeByLlm( packet?: HandoffPacket, escalationHistory?: UserEscalationHistory | null, ): Promise { if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) { return null; } try { const result = await this.llmClient.chatCompletion( [ { content: buildSupervisorSystemPrompt( this.skillRegistry, packet, escalationHistory, ), role: "system", }, { content: buildSupervisorUserPrompt(packet), role: "user", }, ], { jsonMode: true, maxTokens: 150, temperature: 0, tier: 0, }, ); if (!result?.content.trim()) { return null; } const parsed = supervisorRoutingSchema.safeParse( JSON.parse(result.content) as unknown, ); if (!parsed.success || !this.isKnownTarget(parsed.data.target)) { return null; } return parsed.data; } catch (error) { getLogger().warn( { err: error }, "[Supervisor] LLM routing failed, falling back to rules", ); return null; } } private isKnownTarget(target: string): boolean { return ( target === "human-review" || Boolean(this.skillRegistry?.get(target)) ); } private async loadEscalationHistory(userId: string) { if (!this.escalationHistoryService) { return null; } try { return await this.escalationHistoryService.load(userId); } catch (error) { getLogger().warn( { err: error }, "[Supervisor] Failed to load escalation history", ); return null; } } }