Files
lolympus-agent/src/skills/moderation/eval/propose-adapter.ts
T

212 lines
5.5 KiB
TypeScript

import type { ConversationMessage } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import {
loadPromptVariantManifest,
type PromptLayerDiff,
type PromptVariantManifest,
} from "@eval/prompt-variant";
import type { EvalArchiveEntry } from "@eval/archive";
import type { SliceKey } from "@eval/aggregator";
import {
basicPromptSnapshot,
getRequestText,
getVariantManifest,
type SkillProposeAdapter,
} from "../../../eval/loop/skill-propose-adapter.js";
import {
classifyModerationIntent,
type ModerationIntent,
type ModerationIntentKind,
} from "../intents.js";
import { MODERATION_PROMPT_DATA } from "../prompt-data.js";
interface ModerationDerivedSnapshot {
allowShortResponse: boolean;
intent: string;
phase: string;
responseMode: "skip" | "template" | "llm" | "llm-with-tools";
shouldRespond: boolean;
suggestedMaxTokens: number;
tier: 0 | 1 | 2;
}
function formatHistory(history: ConversationMessage[]): string {
if (history.length === 0) {
return "(none)";
}
return history
.slice(-5)
.map((message) => `${message.role}: ${message.content}`)
.join("\n");
}
function formatExamples(intent: ModerationIntent): string {
return MODERATION_PROMPT_DATA.examples[intent.kind]
.slice(0, 2)
.map((example) => `user: ${example.user}\nassistant: ${example.assistant}`)
.join("\n\n");
}
function applyDiff(
layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>,
diff: PromptLayerDiff,
): void {
const target = diff.layer === "tool-policy" ? "policy" : diff.layer;
if (target !== "context" &&
target !== "examples" &&
target !== "persona" &&
target !== "policy" &&
target !== "state") {
return;
}
if (diff.op === "remove") {
layers[target] = "";
return;
}
const text = diff.text?.trim();
if (!text) {
return;
}
if (diff.op === "replace") {
layers[target] = text;
return;
}
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
}
function getManifest(
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): PromptVariantManifest | null {
const manifest = getVariantManifest(variantId, manifests);
if (!manifest) {
return null;
}
return typeof manifest === "string"
? loadPromptVariantManifest(manifest)
: manifest;
}
function buildMessages(
entry: Pick<EvalArchiveEntry, "request">,
variant?: PromptVariantManifest | null,
): ChatMessage[] {
const classified = classifyModerationIntent(getRequestText(entry.request));
const intent = classified.intent;
const layers = {
persona: MODERATION_PROMPT_DATA.persona.join("\n"),
policy: [
...MODERATION_PROMPT_DATA.policy.base,
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
].join("\n"),
state: ["[intent]", intent.kind].join("\n"),
examples: formatExamples(intent),
context: "",
};
for (const diff of variant?.diff ?? []) {
applyDiff(layers, diff);
}
return [
{
role: "system",
content: [
layers.persona,
"",
"[policy]",
layers.policy,
"",
layers.state,
"",
"[examples]",
layers.examples,
...(layers.context ? ["", "[context]", layers.context] : []),
].join("\n"),
},
{
role: "user",
content: [
"[history]",
formatHistory([]),
"",
"[message]",
getRequestText(entry.request),
].join("\n"),
},
];
}
function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot {
return {
allowShortResponse: true,
intent: slice.intent,
phase: slice.phase,
responseMode: slice.intent === "off-topic" ? "template" : "llm",
shouldRespond: slice.intent !== "off-topic",
suggestedMaxTokens: 192,
tier: slice.tier,
};
}
function getSuggestedMaxTokens(entry: EvalArchiveEntry): number {
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
return typeof derived.suggestedMaxTokens === "number"
? derived.suggestedMaxTokens
: 192;
}
function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 {
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
return derived.tier === 0 || derived.tier === 1 || derived.tier === 2
? derived.tier
: entry.tier;
}
export const moderationProposeAdapter: SkillProposeAdapter = {
skill: "moderation",
buildSnapshot(slice, archiveEntry) {
const request = archiveEntry?.request ?? {
requestId: "proposal-snapshot",
skill: "moderation",
chatroomId: "proposal-room",
userId: "proposal-user",
nickname: "문의자",
trigger: "user-message" as const,
userMessage: `slice=${slice.intent}/${slice.phase}`,
};
return basicPromptSnapshot(buildMessages({ request }));
},
derivedFromSlice,
async generateResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string> {
const response = await llmClient.chatCompletion(
buildMessages(entry, getManifest(variantId, manifests)),
{
maxTokens: getSuggestedMaxTokens(entry),
temperature: 0,
tier: getTier(entry),
},
);
return response?.content.trim() ?? "";
},
};
export function normalizeModerationIntentKind(value: string): ModerationIntentKind {
return value in MODERATION_PROMPT_DATA.examples
? (value as ModerationIntentKind)
: "off-topic";
}