Make eval pipeline skill-aware and validate SKILL manifests

This commit is contained in:
dahyeon committed 2026-05-18 16:26:31 -07:00
1 parent 12e731a428
commit 54bc295ead
35 files changed
+1280 -499

No files matched your search

@@ -0,0 +1,211 @@
import type { ConversationMessage } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import {
loadPromptVariantManifest,
type PromptLayerDiff,
type PromptVariantManifest,
} from "@eval/prompt-variant";
import type { EvalArchiveEntry } from "@eval/archive";
import type { SliceKey } from "@eval/aggregator";
import {
basicPromptSnapshot,
getRequestText,
getVariantManifest,
type SkillProposeAdapter,
} from "../../../eval/loop/skill-propose-adapter.js";
import {
classifyModerationIntent,
type ModerationIntent,
type ModerationIntentKind,
} from "../intents.js";
import { MODERATION_PROMPT_DATA } from "../prompt-data.js";
interface ModerationDerivedSnapshot {
allowShortResponse: boolean;
intent: string;
phase: string;
responseMode: "skip" | "template" | "llm" | "llm-with-tools";
shouldRespond: boolean;
suggestedMaxTokens: number;
tier: 0 | 1 | 2;
}
function formatHistory(history: ConversationMessage[]): string {
if (history.length === 0) {
return "(none)";
}
return history
.slice(-5)
.map((message) => `${message.role}: ${message.content}`)
.join("\n");
}
function formatExamples(intent: ModerationIntent): string {
return MODERATION_PROMPT_DATA.examples[intent.kind]
.slice(0, 2)
.map((example) => `user: ${example.user}\nassistant: ${example.assistant}`)
.join("\n\n");
}
function applyDiff(
layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>,
diff: PromptLayerDiff,
): void {
const target = diff.layer === "tool-policy" ? "policy" : diff.layer;
if (target !== "context" &&
target !== "examples" &&
target !== "persona" &&
target !== "policy" &&
target !== "state") {
return;
}
if (diff.op === "remove") {
layers[target] = "";
return;
}
const text = diff.text?.trim();
if (!text) {
return;
}
if (diff.op === "replace") {
layers[target] = text;
return;
}
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
}
function getManifest(
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): PromptVariantManifest | null {
const manifest = getVariantManifest(variantId, manifests);
if (!manifest) {
return null;
}
return typeof manifest === "string"
? loadPromptVariantManifest(manifest)
: manifest;
}
function buildMessages(
entry: Pick<EvalArchiveEntry, "request">,
variant?: PromptVariantManifest | null,
): ChatMessage[] {
const classified = classifyModerationIntent(getRequestText(entry.request));
const intent = classified.intent;
const layers = {
persona: MODERATION_PROMPT_DATA.persona.join("\n"),
policy: [
...MODERATION_PROMPT_DATA.policy.base,
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
].join("\n"),
state: ["[intent]", intent.kind].join("\n"),
examples: formatExamples(intent),
context: "",
};
for (const diff of variant?.diff ?? []) {
applyDiff(layers, diff);
}
return [
{
role: "system",
content: [
layers.persona,
"",
"[policy]",
layers.policy,
"",
layers.state,
"",
"[examples]",
layers.examples,
...(layers.context ? ["", "[context]", layers.context] : []),
].join("\n"),
},
{
role: "user",
content: [
"[history]",
formatHistory([]),
"",
"[message]",
getRequestText(entry.request),
].join("\n"),
},
];
}
function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot {
return {
allowShortResponse: true,
intent: slice.intent,
phase: slice.phase,
responseMode: slice.intent === "off-topic" ? "template" : "llm",
shouldRespond: slice.intent !== "off-topic",
suggestedMaxTokens: 192,
tier: slice.tier,
};
}
function getSuggestedMaxTokens(entry: EvalArchiveEntry): number {
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
return typeof derived.suggestedMaxTokens === "number"
? derived.suggestedMaxTokens
: 192;
}
function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 {
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
return derived.tier === 0 || derived.tier === 1 || derived.tier === 2
? derived.tier
: entry.tier;
}
export const moderationProposeAdapter: SkillProposeAdapter = {
skill: "moderation",
buildSnapshot(slice, archiveEntry) {
const request = archiveEntry?.request ?? {
requestId: "proposal-snapshot",
skill: "moderation",
chatroomId: "proposal-room",
userId: "proposal-user",
nickname: "문의자",
trigger: "user-message" as const,
userMessage: `slice=${slice.intent}/${slice.phase}`,
};
return basicPromptSnapshot(buildMessages({ request }));
},
derivedFromSlice,
async generateResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string> {
const response = await llmClient.chatCompletion(
buildMessages(entry, getManifest(variantId, manifests)),
{
maxTokens: getSuggestedMaxTokens(entry),
temperature: 0,
tier: getTier(entry),
},
);
return response?.content.trim() ?? "";
},
};
export function normalizeModerationIntentKind(value: string): ModerationIntentKind {
return value in MODERATION_PROMPT_DATA.examples
? (value as ModerationIntentKind)
: "off-topic";
}
+12
View File
@@ -0,0 +1,12 @@
import { MODERATION_INTENT_TABLE } from "./intents.js";
import { createModerationToolRegistry } from "./tools.js";
export function getIntentKinds(): string[] {
return Object.keys(MODERATION_INTENT_TABLE);
}
export function getToolNames(): string[] {
return createModerationToolRegistry()
.list()
.map((tool) => tool.name);
}
@@ -0,0 +1,99 @@
import type { LlmClient } from "@llm/llm-client";
import type { PromptVariantManifest } from "@eval/prompt-variant";
import type { EvalArchiveEntry } from "@eval/archive";
import type { SliceKey } from "@eval/aggregator";
import {
getVariantManifest,
type SkillProposeAdapter,
} from "../../../eval/loop/skill-propose-adapter.js";
import {
PromptBuilder,
} from "../prompts/builder.js";
import type { DerivedState, Intent, OnboardingPhase } from "../state/onboarding-state.js";
import type { Blackboard, PromptContext } from "../types.js";
function emptyBlackboard(): Blackboard {
return {
flaggedIssues: [],
openQuestions: [],
recentNicknameAttempts: [],
toneMemory: {},
};
}
function derivedFromSlice(slice: SliceKey): DerivedState {
return {
allowShortResponse: false,
intent: { kind: slice.intent } as Intent,
phase: { kind: slice.phase } as OnboardingPhase,
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 384,
tier: slice.tier,
};
}
function defaultPromptContext(userMessage?: string): PromptContext {
return {
blackboard: emptyBlackboard(),
conversationHistory: [],
userMessage:
userMessage?.trim() ||
"현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
};
}
function getDerived(entry: EvalArchiveEntry, slice?: SliceKey): DerivedState {
const derived = entry.derived as Partial<DerivedState>;
if (derived.intent && derived.phase) {
return derived as DerivedState;
}
return derivedFromSlice(
slice ?? {
skill: "onboarding",
tier: entry.tier,
intent: "unknown",
phase: "unknown",
},
);
}
export const onboardingProposeAdapter: SkillProposeAdapter = {
skill: "onboarding",
buildSnapshot(slice, archiveEntry) {
return PromptBuilder.forDerived(
archiveEntry ? getDerived(archiveEntry, slice) : derivedFromSlice(slice),
null,
defaultPromptContext(archiveEntry?.request.userMessage),
).buildSystemPrompt();
},
derivedFromSlice,
async generateResponse(
llmClient: LlmClient,
entry: EvalArchiveEntry,
variantId: string,
manifests?: Record<string, PromptVariantManifest>,
): Promise<string> {
let builder = PromptBuilder.forDerived(
getDerived(entry),
null,
defaultPromptContext(entry.request.userMessage),
);
const manifest = getVariantManifest(variantId, manifests);
if (manifest) {
builder = builder.applyVariant(manifest);
}
const derived = getDerived(entry);
const response = await llmClient.chatCompletion(builder.build(), {
maxTokens: derived.suggestedMaxTokens,
temperature: 0,
tier: derived.tier,
});
return response?.content.trim() ?? "";
},
};
+12
View File
@@ -0,0 +1,12 @@
import { ONBOARDING_INTENT_TABLE } from "./intents.js";
import { createOnboardingToolRegistry } from "./tools.js";
export function getIntentKinds(): string[] {
return Object.keys(ONBOARDING_INTENT_TABLE);
}
export function getToolNames(): string[] {
return createOnboardingToolRegistry()
.list()
.map((tool) => tool.name);
}
+12 -67
View File
@@ -1,6 +1,11 @@
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import {
loadPromptVariantManifest,
type PromptLayerDiff,
type PromptLayerName,
type PromptVariantManifest,
} from "@eval/prompt-variant";
import { SkillRegistry } from "@skills/skill-loader";
import type { ChatMessage } from "@llm/types";
import {
@@ -19,35 +24,12 @@ import {
import { buildToolPolicy } from "./tool-policy.js";
import { defaultExampleRetriever } from "./example-store.js";
export type PromptLayerName =
| "context"
| "examples"
| "persona"
| "policy"
| "state"
| "tool-policy";
export type PromptLayerOperation = "append" | "remove" | "replace";
export interface PromptLayerDiff {
layer: PromptLayerName;
op: PromptLayerOperation;
reason?: string | undefined;
text?: string | undefined;
}
export interface PromptVariantManifest {
description?: string | undefined;
diff: PromptLayerDiff[];
expectedImprovement?: string | undefined;
rationale?: string | undefined;
requiredResources?: string[] | undefined;
requiredTools?: string[] | undefined;
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
variantId: string;
}
const variantCache = new Map<string, PromptVariantManifest | null>();
export {
loadPromptVariantManifest,
type PromptLayerDiff,
type PromptLayerName,
type PromptVariantManifest,
} from "@eval/prompt-variant";
function getSkillRootDir(): string {
return resolve(process.cwd(), "src", "skills");
@@ -85,43 +67,6 @@ function applyLayerDiff(
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
}
export function loadPromptVariantManifest(
variantId: string,
): PromptVariantManifest | null {
const normalized = variantId.trim();
if (!normalized) {
return null;
}
if (variantCache.has(normalized)) {
return variantCache.get(normalized) ?? null;
}
const path = resolve(
process.cwd(),
"src",
"eval",
"proposals",
`${normalized}.json`,
);
if (!existsSync(path)) {
variantCache.set(normalized, null);
return null;
}
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
variantCache.set(normalized, null);
return null;
}
variantCache.set(normalized, parsed);
return parsed;
} catch {
variantCache.set(normalized, null);
return null;
}
}
function skillIndexLayer(registry: SkillRegistry): string {
const lines = registry.list().map(
(manifest) =>
+3
View File
@@ -189,6 +189,7 @@ export class OnboardingSkill implements Skill {
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
return response;
}
@@ -222,6 +223,7 @@ export class OnboardingSkill implements Skill {
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
return response;
@@ -294,6 +296,7 @@ export class OnboardingSkill implements Skill {
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
}
-29
View File
@@ -1,6 +1,3 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { checkOnboardingProgressTool } from "@tools/onboarding/check-onboarding-progress";
import { getRoomContextTool } from "@tools/onboarding/get-room-context";
import { lookupCommunityRuleTool } from "@tools/onboarding/lookup-community-rule";
@@ -17,32 +14,6 @@ const ONBOARDING_TOOLS = [
verifyDiscordNicknameTool,
] as const satisfies readonly Tool<unknown, unknown>[];
function readSkillManifestToolNames(): string[] {
const skillPath = resolve(process.cwd(), "src", "skills", "onboarding", "SKILL.md");
const raw = readFileSync(skillPath, "utf8");
const match = raw.match(/^tools:\s*\[([^\]]*)\]/mu);
if (!match?.[1]) {
return [];
}
return match[1]
.split(",")
.map((item) => item.trim())
.filter(Boolean);
}
function assertToolManifestMatchesRegistry(): void {
const manifestNames = readSkillManifestToolNames().sort();
const registryNames = ONBOARDING_TOOLS.map((tool) => tool.name).sort();
if (manifestNames.join("\n") !== registryNames.join("\n")) {
throw new Error(
`Onboarding SKILL.md tools do not match registry: manifest=${manifestNames.join(",")} registry=${registryNames.join(",")}`,
);
}
}
assertToolManifestMatchesRegistry();
export function createOnboardingToolRegistry(): ToolRegistry {
const registry = new ToolRegistry();
for (const tool of ONBOARDING_TOOLS) {
+106
View File
@@ -0,0 +1,106 @@
import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
import * as moderationBinding from "./moderation/manifest-binding.js";
import * as onboardingBinding from "./onboarding/manifest-binding.js";
interface ManifestBinding {
getIntentKinds(): string[];
getToolNames(): string[];
}
const BINDINGS: Record<string, ManifestBinding> = {
moderation: moderationBinding,
onboarding: onboardingBinding,
};
function sortUnique(values: readonly string[]): string[] {
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
}
function format(values: readonly string[]): string {
return values.length === 0 ? "(none)" : values.join(", ");
}
function diff(left: readonly string[], right: readonly string[]): string[] {
const rightSet = new Set(right);
return left.filter((item) => !rightSet.has(item));
}
function validateManifest(
manifest: SkillManifest,
binding: ManifestBinding | undefined,
): string[] {
if (!binding) {
return [`${manifest.name}: missing manifest binding`];
}
const errors: string[] = [];
const manifestIntents = sortUnique(manifest.intents);
const codeIntents = sortUnique(binding.getIntentKinds());
const unknownIntents = diff(manifestIntents, codeIntents);
if (unknownIntents.length > 0) {
errors.push(
`${manifest.name}: SKILL.md intents not found in code: ${format(unknownIntents)}`,
);
}
const manifestTools = sortUnique(manifest.tools);
const codeTools = sortUnique(binding.getToolNames());
const missingTools = diff(manifestTools, codeTools);
const undocumentedTools = diff(codeTools, manifestTools);
if (missingTools.length > 0 || undocumentedTools.length > 0) {
errors.push(
`${manifest.name}: SKILL.md tools mismatch; missingInRegistry=${format(missingTools)} missingInManifest=${format(undocumentedTools)}`,
);
}
for (const [key, path] of Object.entries(manifest.resources)) {
if (!existsSync(path)) {
errors.push(`${manifest.name}: resource not found ${key} -> ${path}`);
}
}
return errors;
}
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
const byPriority = new Map<number, string[]>();
for (const manifest of manifests) {
const current = byPriority.get(manifest.priority) ?? [];
current.push(manifest.name);
byPriority.set(manifest.priority, current);
}
return [...byPriority.entries()].flatMap(([priority, names]) =>
names.length > 1
? [`priority=${priority} is shared by skills: ${format(names.sort())}`]
: [],
);
}
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
const registry = SkillRegistry.load(rootDir);
const manifests = registry.list();
return [
...manifests.flatMap((manifest) =>
validateManifest(manifest, BINDINGS[manifest.name]),
),
...validatePriorityConflicts(manifests),
];
}
const invokedPath = process.argv[1] ? resolve(process.argv[1]) : "";
if (invokedPath === resolve(fileURLToPath(import.meta.url))) {
const errors = validateSkillManifests();
if (errors.length > 0) {
for (const error of errors) {
console.error(`[skills] ${error}`);
}
process.exitCode = 1;
} else {
console.info("[skills] manifests ok");
}
}