import type { ConversationMessage } from "@agent/types"; import type { LlmClient } from "@llm/llm-client"; import type { ChatMessage } from "@llm/types"; import { loadPromptVariantManifest, type PromptLayerDiff, type PromptVariantManifest, } from "@eval/prompt-variant"; import type { EvalArchiveEntry } from "@eval/archive"; import type { SliceKey } from "@eval/aggregator"; import { basicPromptSnapshot, getRequestText, getVariantManifest, type SkillProposeAdapter, } from "../../../eval/loop/skill-propose-adapter.js"; import { classifyModerationIntent, type ModerationIntent, type ModerationIntentKind, } from "../intents.js"; import { MODERATION_PROMPT_DATA } from "../prompt-data.js"; interface ModerationDerivedSnapshot { allowShortResponse: boolean; intent: string; phase: string; responseMode: "skip" | "template" | "llm" | "llm-with-tools"; shouldRespond: boolean; suggestedMaxTokens: number; tier: 0 | 1 | 2; } function formatHistory(history: ConversationMessage[]): string { if (history.length === 0) { return "(none)"; } return history .slice(-5) .map((message) => `${message.role}: ${message.content}`) .join("\n"); } function formatExamples(intent: ModerationIntent): string { return MODERATION_PROMPT_DATA.examples[intent.kind] .slice(0, 2) .map((example) => `user: ${example.user}\nassistant: ${example.assistant}`) .join("\n\n"); } function applyDiff( layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>, diff: PromptLayerDiff, ): void { const target = diff.layer === "tool-policy" ? "policy" : diff.layer; if (target !== "context" && target !== "examples" && target !== "persona" && target !== "policy" && target !== "state") { return; } if (diff.op === "remove") { layers[target] = ""; return; } const text = diff.text?.trim(); if (!text) { return; } if (diff.op === "replace") { layers[target] = text; return; } layers[target] = layers[target] ? `${layers[target]}\n${text}` : text; } function getManifest( variantId: string, manifests?: Record, ): PromptVariantManifest | null { const manifest = getVariantManifest(variantId, manifests); if (!manifest) { return null; } return typeof manifest === "string" ? loadPromptVariantManifest(manifest) : manifest; } function buildMessages( entry: Pick, variant?: PromptVariantManifest | null, ): ChatMessage[] { const classified = classifyModerationIntent(getRequestText(entry.request)); const intent = classified.intent; const layers = { persona: MODERATION_PROMPT_DATA.persona.join("\n"), policy: [ ...MODERATION_PROMPT_DATA.policy.base, ...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind], ].join("\n"), state: ["[intent]", intent.kind].join("\n"), examples: formatExamples(intent), context: "", }; for (const diff of variant?.diff ?? []) { applyDiff(layers, diff); } return [ { role: "system", content: [ layers.persona, "", "[policy]", layers.policy, "", layers.state, "", "[examples]", layers.examples, ...(layers.context ? ["", "[context]", layers.context] : []), ].join("\n"), }, { role: "user", content: [ "[history]", formatHistory([]), "", "[message]", getRequestText(entry.request), ].join("\n"), }, ]; } function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot { return { allowShortResponse: true, intent: slice.intent, phase: slice.phase, responseMode: slice.intent === "off-topic" ? "template" : "llm", shouldRespond: slice.intent !== "off-topic", suggestedMaxTokens: 192, tier: slice.tier, }; } function getSuggestedMaxTokens(entry: EvalArchiveEntry): number { const derived = entry.derived as Partial; return typeof derived.suggestedMaxTokens === "number" ? derived.suggestedMaxTokens : 192; } function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 { const derived = entry.derived as Partial; return derived.tier === 0 || derived.tier === 1 || derived.tier === 2 ? derived.tier : entry.tier; } export const moderationProposeAdapter: SkillProposeAdapter = { skill: "moderation", buildSnapshot(slice, archiveEntry) { const request = archiveEntry?.request ?? { requestId: "proposal-snapshot", skill: "moderation", chatroomId: "proposal-room", userId: "proposal-user", nickname: "문의자", trigger: "user-message" as const, userMessage: `slice=${slice.intent}/${slice.phase}`, }; return basicPromptSnapshot(buildMessages({ request })); }, derivedFromSlice, async generateResponse( llmClient: LlmClient, entry: EvalArchiveEntry, variantId: string, manifests?: Record, ): Promise { const response = await llmClient.chatCompletion( buildMessages(entry, getManifest(variantId, manifests)), { maxTokens: getSuggestedMaxTokens(entry), temperature: 0, tier: getTier(entry), }, ); return response?.content.trim() ?? ""; }, }; export function normalizeModerationIntentKind(value: string): ModerationIntentKind { return value in MODERATION_PROMPT_DATA.examples ? (value as ModerationIntentKind) : "off-topic"; }