Files
lolympus-agent/src/skills/supervisor/skill.ts
T

256 lines
7.2 KiB
TypeScript

import type {
AgentRequest,
AgentResponse,
ConversationMessage,
} from "@agent/types";
import { readHandoffPacket } from "@agent/handoff-packet";
import type { HandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import type { LlmClient } from "@llm/llm-client";
import type {
EscalationHistoryService,
UserEscalationHistory,
} from "../../memory/escalation-history.js";
import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics";
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
import { getLogger } from "@obs/logger";
import {
getSupervisorLlmConfidenceThreshold,
getSupervisorLlmEnabled,
getSupervisorLlmResponseEnabled,
} from "../../config.js";
import {
buildSupervisorSystemPrompt,
buildSupervisorUserPrompt,
} from "./capability-prompt.js";
import { appendSupervisorReview } from "./review-queue.js";
import { routeSupervisorTarget } from "./routing-rules.js";
import {
supervisorRoutingSchema,
type SupervisorRoutingDecision,
} from "./schema.js";
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: ["*"],
cannotHandle: [],
domain: "meta",
handoffTargets: ["onboarding", "moderation"],
outputContract: {
format: "text",
maxChars: "500",
requiredFields: "text",
},
riskTier: "medium",
};
const TEMPLATE_HUMAN_REVIEW_REPLY =
"확인이 필요한 내용이라 운영자에게 전달했어요.";
const UNSAFE_USER_MESSAGE_PATTERNS = [
/https?:\/\//iu,
/www\./iu,
/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/iu,
/\b\d{2,3}[-.\s]?\d{3,4}[-.\s]?\d{4}\b/u,
/\[(?:NO_RESPONSE|HANDOFF(?::[^\]]*)?)\]/iu,
] as const;
function getSafeSupervisorUserMessage(
decision: SupervisorRoutingDecision | null,
): {
source: "llm" | "template";
text: string;
} {
const message = decision?.userMessage?.trim();
if (
!getSupervisorLlmResponseEnabled() ||
!message ||
message.length < 10 ||
message.length > 400 ||
UNSAFE_USER_MESSAGE_PATTERNS.some((pattern) => pattern.test(message))
) {
return { source: "template", text: TEMPLATE_HUMAN_REVIEW_REPLY };
}
return { source: "llm", text: message };
}
export class SupervisorSkill implements Skill {
readonly name = "supervisor";
readonly priority = 10;
private readonly alertService: AlertService;
constructor(
private readonly garnetClient: Pick<
GarnetClient,
"appendStream" | "setStringIfAbsent"
>,
natsClient: Pick<NatsClient, "publish">,
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
private readonly skillRegistry?: SkillRegistry | undefined,
private readonly escalationHistoryService?: EscalationHistoryService | undefined,
) {
this.alertService = new AlertService(
garnetClient,
natsClient,
createSlackNotifierFromEnv(),
);
}
canHandle(request: AgentRequest): boolean {
return request.skill === this.name;
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
_conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const packet = readHandoffPacket(request);
const escalationHistory = await this.loadEscalationHistory(request.userId);
const llmDecision = await this.routeByLlm(packet, escalationHistory);
const useLlmRoute =
llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
const target = useLlmRoute
? llmDecision.target
: routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope";
const queuedAt = new Date().toISOString();
const reviewId = await appendSupervisorReview(this.garnetClient, {
chatroomId: request.chatroomId,
packet: JSON.stringify(packet ?? null),
queuedAt,
reason,
requestId: request.requestId,
target: target ?? "human-review",
userId: request.userId,
});
await this.escalationHistoryService?.recordEscalation({
chatroomId: request.chatroomId,
packetSummary: packet?.userGoal ?? packet?.summary ?? "",
queuedAt,
reason,
reviewId,
sourceChain: packet?.sourceChain ?? [],
target: target ?? "human-review",
userId: request.userId,
});
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
await this.alertService.sendAlert(
{
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
severity: "warn",
skill: this.name,
suggestedAction: "Review the supervisor handoff queue.",
threshold: 0,
timestamp: new Date().toISOString(),
value: 1,
window: "request",
},
{
dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`,
},
);
const userMessage = getSafeSupervisorUserMessage(
useLlmRoute ? llmDecision : null,
);
return {
confidence: 0.6,
metadata: {
routedBy: useLlmRoute ? "llm" : "rules",
source: userMessage.source,
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
suggestedTarget: target,
},
requestId: request.requestId,
source: "template",
text: userMessage.text,
};
}
private async routeByLlm(
packet?: HandoffPacket,
escalationHistory?: UserEscalationHistory | null,
): Promise<SupervisorRoutingDecision | null> {
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
return null;
}
try {
const result = await this.llmClient.chatCompletion(
[
{
content: buildSupervisorSystemPrompt(
this.skillRegistry,
packet,
escalationHistory,
),
role: "system",
},
{
content: buildSupervisorUserPrompt(packet),
role: "user",
},
],
{
jsonMode: true,
maxTokens: 150,
temperature: 0,
tier: 0,
},
);
if (!result?.content.trim()) {
return null;
}
const parsed = supervisorRoutingSchema.safeParse(
JSON.parse(result.content) as unknown,
);
if (!parsed.success || !this.isKnownTarget(parsed.data.target)) {
return null;
}
return parsed.data;
} catch (error) {
getLogger().warn(
{ err: error },
"[Supervisor] LLM routing failed, falling back to rules",
);
return null;
}
}
private isKnownTarget(target: string): boolean {
return (
target === "human-review" || Boolean(this.skillRegistry?.get(target))
);
}
private async loadEscalationHistory(userId: string) {
if (!this.escalationHistoryService) {
return null;
}
try {
return await this.escalationHistoryService.load(userId);
} catch (error) {
getLogger().warn(
{ err: error },
"[Supervisor] Failed to load escalation history",
);
return null;
}
}
}