Make eval pipeline skill-aware and validate SKILL manifests
This commit is contained in:
1 parent
12e731a428
commit
54bc295ead
35 files changed
+1280
-499
No files matched your search
@@ -0,0 +1,211 @@
|
||||
import type { ConversationMessage } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import {
|
||||
loadPromptVariantManifest,
|
||||
type PromptLayerDiff,
|
||||
type PromptVariantManifest,
|
||||
} from "@eval/prompt-variant";
|
||||
import type { EvalArchiveEntry } from "@eval/archive";
|
||||
import type { SliceKey } from "@eval/aggregator";
|
||||
import {
|
||||
basicPromptSnapshot,
|
||||
getRequestText,
|
||||
getVariantManifest,
|
||||
type SkillProposeAdapter,
|
||||
} from "../../../eval/loop/skill-propose-adapter.js";
|
||||
import {
|
||||
classifyModerationIntent,
|
||||
type ModerationIntent,
|
||||
type ModerationIntentKind,
|
||||
} from "../intents.js";
|
||||
import { MODERATION_PROMPT_DATA } from "../prompt-data.js";
|
||||
|
||||
interface ModerationDerivedSnapshot {
|
||||
allowShortResponse: boolean;
|
||||
intent: string;
|
||||
phase: string;
|
||||
responseMode: "skip" | "template" | "llm" | "llm-with-tools";
|
||||
shouldRespond: boolean;
|
||||
suggestedMaxTokens: number;
|
||||
tier: 0 | 1 | 2;
|
||||
}
|
||||
|
||||
function formatHistory(history: ConversationMessage[]): string {
|
||||
if (history.length === 0) {
|
||||
return "(none)";
|
||||
}
|
||||
|
||||
return history
|
||||
.slice(-5)
|
||||
.map((message) => `${message.role}: ${message.content}`)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function formatExamples(intent: ModerationIntent): string {
|
||||
return MODERATION_PROMPT_DATA.examples[intent.kind]
|
||||
.slice(0, 2)
|
||||
.map((example) => `user: ${example.user}\nassistant: ${example.assistant}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function applyDiff(
|
||||
layers: Record<"context" | "examples" | "persona" | "policy" | "state", string>,
|
||||
diff: PromptLayerDiff,
|
||||
): void {
|
||||
const target = diff.layer === "tool-policy" ? "policy" : diff.layer;
|
||||
if (target !== "context" &&
|
||||
target !== "examples" &&
|
||||
target !== "persona" &&
|
||||
target !== "policy" &&
|
||||
target !== "state") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (diff.op === "remove") {
|
||||
layers[target] = "";
|
||||
return;
|
||||
}
|
||||
|
||||
const text = diff.text?.trim();
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
if (diff.op === "replace") {
|
||||
layers[target] = text;
|
||||
return;
|
||||
}
|
||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||
}
|
||||
|
||||
function getManifest(
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): PromptVariantManifest | null {
|
||||
const manifest = getVariantManifest(variantId, manifests);
|
||||
if (!manifest) {
|
||||
return null;
|
||||
}
|
||||
return typeof manifest === "string"
|
||||
? loadPromptVariantManifest(manifest)
|
||||
: manifest;
|
||||
}
|
||||
|
||||
function buildMessages(
|
||||
entry: Pick<EvalArchiveEntry, "request">,
|
||||
variant?: PromptVariantManifest | null,
|
||||
): ChatMessage[] {
|
||||
const classified = classifyModerationIntent(getRequestText(entry.request));
|
||||
const intent = classified.intent;
|
||||
const layers = {
|
||||
persona: MODERATION_PROMPT_DATA.persona.join("\n"),
|
||||
policy: [
|
||||
...MODERATION_PROMPT_DATA.policy.base,
|
||||
...MODERATION_PROMPT_DATA.policy.perIntent[intent.kind],
|
||||
].join("\n"),
|
||||
state: ["[intent]", intent.kind].join("\n"),
|
||||
examples: formatExamples(intent),
|
||||
context: "",
|
||||
};
|
||||
|
||||
for (const diff of variant?.diff ?? []) {
|
||||
applyDiff(layers, diff);
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
layers.persona,
|
||||
"",
|
||||
"[policy]",
|
||||
layers.policy,
|
||||
"",
|
||||
layers.state,
|
||||
"",
|
||||
"[examples]",
|
||||
layers.examples,
|
||||
...(layers.context ? ["", "[context]", layers.context] : []),
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
"[history]",
|
||||
formatHistory([]),
|
||||
"",
|
||||
"[message]",
|
||||
getRequestText(entry.request),
|
||||
].join("\n"),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function derivedFromSlice(slice: SliceKey): ModerationDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: true,
|
||||
intent: slice.intent,
|
||||
phase: slice.phase,
|
||||
responseMode: slice.intent === "off-topic" ? "template" : "llm",
|
||||
shouldRespond: slice.intent !== "off-topic",
|
||||
suggestedMaxTokens: 192,
|
||||
tier: slice.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function getSuggestedMaxTokens(entry: EvalArchiveEntry): number {
|
||||
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
|
||||
return typeof derived.suggestedMaxTokens === "number"
|
||||
? derived.suggestedMaxTokens
|
||||
: 192;
|
||||
}
|
||||
|
||||
function getTier(entry: EvalArchiveEntry): 0 | 1 | 2 {
|
||||
const derived = entry.derived as Partial<ModerationDerivedSnapshot>;
|
||||
return derived.tier === 0 || derived.tier === 1 || derived.tier === 2
|
||||
? derived.tier
|
||||
: entry.tier;
|
||||
}
|
||||
|
||||
export const moderationProposeAdapter: SkillProposeAdapter = {
|
||||
skill: "moderation",
|
||||
|
||||
buildSnapshot(slice, archiveEntry) {
|
||||
const request = archiveEntry?.request ?? {
|
||||
requestId: "proposal-snapshot",
|
||||
skill: "moderation",
|
||||
chatroomId: "proposal-room",
|
||||
userId: "proposal-user",
|
||||
nickname: "문의자",
|
||||
trigger: "user-message" as const,
|
||||
userMessage: `slice=${slice.intent}/${slice.phase}`,
|
||||
};
|
||||
return basicPromptSnapshot(buildMessages({ request }));
|
||||
},
|
||||
|
||||
derivedFromSlice,
|
||||
|
||||
async generateResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string> {
|
||||
const response = await llmClient.chatCompletion(
|
||||
buildMessages(entry, getManifest(variantId, manifests)),
|
||||
{
|
||||
maxTokens: getSuggestedMaxTokens(entry),
|
||||
temperature: 0,
|
||||
tier: getTier(entry),
|
||||
},
|
||||
);
|
||||
|
||||
return response?.content.trim() ?? "";
|
||||
},
|
||||
};
|
||||
|
||||
export function normalizeModerationIntentKind(value: string): ModerationIntentKind {
|
||||
return value in MODERATION_PROMPT_DATA.examples
|
||||
? (value as ModerationIntentKind)
|
||||
: "off-topic";
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { MODERATION_INTENT_TABLE } from "./intents.js";
|
||||
import { createModerationToolRegistry } from "./tools.js";
|
||||
|
||||
export function getIntentKinds(): string[] {
|
||||
return Object.keys(MODERATION_INTENT_TABLE);
|
||||
}
|
||||
|
||||
export function getToolNames(): string[] {
|
||||
return createModerationToolRegistry()
|
||||
.list()
|
||||
.map((tool) => tool.name);
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { PromptVariantManifest } from "@eval/prompt-variant";
|
||||
import type { EvalArchiveEntry } from "@eval/archive";
|
||||
import type { SliceKey } from "@eval/aggregator";
|
||||
import {
|
||||
getVariantManifest,
|
||||
type SkillProposeAdapter,
|
||||
} from "../../../eval/loop/skill-propose-adapter.js";
|
||||
import {
|
||||
PromptBuilder,
|
||||
} from "../prompts/builder.js";
|
||||
import type { DerivedState, Intent, OnboardingPhase } from "../state/onboarding-state.js";
|
||||
import type { Blackboard, PromptContext } from "../types.js";
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function derivedFromSlice(slice: SliceKey): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: { kind: slice.intent } as Intent,
|
||||
phase: { kind: slice.phase } as OnboardingPhase,
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 384,
|
||||
tier: slice.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function defaultPromptContext(userMessage?: string): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage:
|
||||
userMessage?.trim() ||
|
||||
"현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
function getDerived(entry: EvalArchiveEntry, slice?: SliceKey): DerivedState {
|
||||
const derived = entry.derived as Partial<DerivedState>;
|
||||
if (derived.intent && derived.phase) {
|
||||
return derived as DerivedState;
|
||||
}
|
||||
return derivedFromSlice(
|
||||
slice ?? {
|
||||
skill: "onboarding",
|
||||
tier: entry.tier,
|
||||
intent: "unknown",
|
||||
phase: "unknown",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export const onboardingProposeAdapter: SkillProposeAdapter = {
|
||||
skill: "onboarding",
|
||||
|
||||
buildSnapshot(slice, archiveEntry) {
|
||||
return PromptBuilder.forDerived(
|
||||
archiveEntry ? getDerived(archiveEntry, slice) : derivedFromSlice(slice),
|
||||
null,
|
||||
defaultPromptContext(archiveEntry?.request.userMessage),
|
||||
).buildSystemPrompt();
|
||||
},
|
||||
|
||||
derivedFromSlice,
|
||||
|
||||
async generateResponse(
|
||||
llmClient: LlmClient,
|
||||
entry: EvalArchiveEntry,
|
||||
variantId: string,
|
||||
manifests?: Record<string, PromptVariantManifest>,
|
||||
): Promise<string> {
|
||||
let builder = PromptBuilder.forDerived(
|
||||
getDerived(entry),
|
||||
null,
|
||||
defaultPromptContext(entry.request.userMessage),
|
||||
);
|
||||
const manifest = getVariantManifest(variantId, manifests);
|
||||
if (manifest) {
|
||||
builder = builder.applyVariant(manifest);
|
||||
}
|
||||
|
||||
const derived = getDerived(entry);
|
||||
const response = await llmClient.chatCompletion(builder.build(), {
|
||||
maxTokens: derived.suggestedMaxTokens,
|
||||
temperature: 0,
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
return response?.content.trim() ?? "";
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
import { ONBOARDING_INTENT_TABLE } from "./intents.js";
|
||||
import { createOnboardingToolRegistry } from "./tools.js";
|
||||
|
||||
export function getIntentKinds(): string[] {
|
||||
return Object.keys(ONBOARDING_INTENT_TABLE);
|
||||
}
|
||||
|
||||
export function getToolNames(): string[] {
|
||||
return createOnboardingToolRegistry()
|
||||
.list()
|
||||
.map((tool) => tool.name);
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import {
|
||||
loadPromptVariantManifest,
|
||||
type PromptLayerDiff,
|
||||
type PromptLayerName,
|
||||
type PromptVariantManifest,
|
||||
} from "@eval/prompt-variant";
|
||||
import { SkillRegistry } from "@skills/skill-loader";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import {
|
||||
@@ -19,35 +24,12 @@ import {
|
||||
import { buildToolPolicy } from "./tool-policy.js";
|
||||
import { defaultExampleRetriever } from "./example-store.js";
|
||||
|
||||
export type PromptLayerName =
|
||||
| "context"
|
||||
| "examples"
|
||||
| "persona"
|
||||
| "policy"
|
||||
| "state"
|
||||
| "tool-policy";
|
||||
|
||||
export type PromptLayerOperation = "append" | "remove" | "replace";
|
||||
|
||||
export interface PromptLayerDiff {
|
||||
layer: PromptLayerName;
|
||||
op: PromptLayerOperation;
|
||||
reason?: string | undefined;
|
||||
text?: string | undefined;
|
||||
}
|
||||
|
||||
export interface PromptVariantManifest {
|
||||
description?: string | undefined;
|
||||
diff: PromptLayerDiff[];
|
||||
expectedImprovement?: string | undefined;
|
||||
rationale?: string | undefined;
|
||||
requiredResources?: string[] | undefined;
|
||||
requiredTools?: string[] | undefined;
|
||||
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||
export {
|
||||
loadPromptVariantManifest,
|
||||
type PromptLayerDiff,
|
||||
type PromptLayerName,
|
||||
type PromptVariantManifest,
|
||||
} from "@eval/prompt-variant";
|
||||
|
||||
function getSkillRootDir(): string {
|
||||
return resolve(process.cwd(), "src", "skills");
|
||||
@@ -85,43 +67,6 @@ function applyLayerDiff(
|
||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||
}
|
||||
|
||||
export function loadPromptVariantManifest(
|
||||
variantId: string,
|
||||
): PromptVariantManifest | null {
|
||||
const normalized = variantId.trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (variantCache.has(normalized)) {
|
||||
return variantCache.get(normalized) ?? null;
|
||||
}
|
||||
|
||||
const path = resolve(
|
||||
process.cwd(),
|
||||
"src",
|
||||
"eval",
|
||||
"proposals",
|
||||
`${normalized}.json`,
|
||||
);
|
||||
if (!existsSync(path)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
||||
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
variantCache.set(normalized, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function skillIndexLayer(registry: SkillRegistry): string {
|
||||
const lines = registry.list().map(
|
||||
(manifest) =>
|
||||
|
||||
@@ -189,6 +189,7 @@ export class OnboardingSkill implements Skill {
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
skill: this.name,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
@@ -222,6 +223,7 @@ export class OnboardingSkill implements Skill {
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
skill: this.name,
|
||||
});
|
||||
|
||||
return response;
|
||||
@@ -294,6 +296,7 @@ export class OnboardingSkill implements Skill {
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
skill: this.name,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { checkOnboardingProgressTool } from "@tools/onboarding/check-onboarding-progress";
|
||||
import { getRoomContextTool } from "@tools/onboarding/get-room-context";
|
||||
import { lookupCommunityRuleTool } from "@tools/onboarding/lookup-community-rule";
|
||||
@@ -17,32 +14,6 @@ const ONBOARDING_TOOLS = [
|
||||
verifyDiscordNicknameTool,
|
||||
] as const satisfies readonly Tool<unknown, unknown>[];
|
||||
|
||||
function readSkillManifestToolNames(): string[] {
|
||||
const skillPath = resolve(process.cwd(), "src", "skills", "onboarding", "SKILL.md");
|
||||
const raw = readFileSync(skillPath, "utf8");
|
||||
const match = raw.match(/^tools:\s*\[([^\]]*)\]/mu);
|
||||
if (!match?.[1]) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return match[1]
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function assertToolManifestMatchesRegistry(): void {
|
||||
const manifestNames = readSkillManifestToolNames().sort();
|
||||
const registryNames = ONBOARDING_TOOLS.map((tool) => tool.name).sort();
|
||||
if (manifestNames.join("\n") !== registryNames.join("\n")) {
|
||||
throw new Error(
|
||||
`Onboarding SKILL.md tools do not match registry: manifest=${manifestNames.join(",")} registry=${registryNames.join(",")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
assertToolManifestMatchesRegistry();
|
||||
|
||||
export function createOnboardingToolRegistry(): ToolRegistry {
|
||||
const registry = new ToolRegistry();
|
||||
for (const tool of ONBOARDING_TOOLS) {
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
|
||||
import * as moderationBinding from "./moderation/manifest-binding.js";
|
||||
import * as onboardingBinding from "./onboarding/manifest-binding.js";
|
||||
|
||||
interface ManifestBinding {
|
||||
getIntentKinds(): string[];
|
||||
getToolNames(): string[];
|
||||
}
|
||||
|
||||
const BINDINGS: Record<string, ManifestBinding> = {
|
||||
moderation: moderationBinding,
|
||||
onboarding: onboardingBinding,
|
||||
};
|
||||
|
||||
function sortUnique(values: readonly string[]): string[] {
|
||||
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function format(values: readonly string[]): string {
|
||||
return values.length === 0 ? "(none)" : values.join(", ");
|
||||
}
|
||||
|
||||
function diff(left: readonly string[], right: readonly string[]): string[] {
|
||||
const rightSet = new Set(right);
|
||||
return left.filter((item) => !rightSet.has(item));
|
||||
}
|
||||
|
||||
function validateManifest(
|
||||
manifest: SkillManifest,
|
||||
binding: ManifestBinding | undefined,
|
||||
): string[] {
|
||||
if (!binding) {
|
||||
return [`${manifest.name}: missing manifest binding`];
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
const manifestIntents = sortUnique(manifest.intents);
|
||||
const codeIntents = sortUnique(binding.getIntentKinds());
|
||||
const unknownIntents = diff(manifestIntents, codeIntents);
|
||||
if (unknownIntents.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: SKILL.md intents not found in code: ${format(unknownIntents)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const manifestTools = sortUnique(manifest.tools);
|
||||
const codeTools = sortUnique(binding.getToolNames());
|
||||
const missingTools = diff(manifestTools, codeTools);
|
||||
const undocumentedTools = diff(codeTools, manifestTools);
|
||||
if (missingTools.length > 0 || undocumentedTools.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: SKILL.md tools mismatch; missingInRegistry=${format(missingTools)} missingInManifest=${format(undocumentedTools)}`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const [key, path] of Object.entries(manifest.resources)) {
|
||||
if (!existsSync(path)) {
|
||||
errors.push(`${manifest.name}: resource not found ${key} -> ${path}`);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||
const byPriority = new Map<number, string[]>();
|
||||
for (const manifest of manifests) {
|
||||
const current = byPriority.get(manifest.priority) ?? [];
|
||||
current.push(manifest.name);
|
||||
byPriority.set(manifest.priority, current);
|
||||
}
|
||||
|
||||
return [...byPriority.entries()].flatMap(([priority, names]) =>
|
||||
names.length > 1
|
||||
? [`priority=${priority} is shared by skills: ${format(names.sort())}`]
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
|
||||
const registry = SkillRegistry.load(rootDir);
|
||||
const manifests = registry.list();
|
||||
return [
|
||||
...manifests.flatMap((manifest) =>
|
||||
validateManifest(manifest, BINDINGS[manifest.name]),
|
||||
),
|
||||
...validatePriorityConflicts(manifests),
|
||||
];
|
||||
}
|
||||
|
||||
const invokedPath = process.argv[1] ? resolve(process.argv[1]) : "";
|
||||
if (invokedPath === resolve(fileURLToPath(import.meta.url))) {
|
||||
const errors = validateSkillManifests();
|
||||
if (errors.length > 0) {
|
||||
for (const error of errors) {
|
||||
console.error(`[skills] ${error}`);
|
||||
}
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
console.info("[skills] manifests ok");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user