Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent 6f09408423
commit 41dcfcdef7
71 files changed
+7057 -102

No files matched your search

+54 -1
View File
@@ -1,4 +1,4 @@
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import { PromptBuilder } from "./builder.js";
import type { DerivedState } from "../state/onboarding-state.js";
@@ -60,6 +60,11 @@ function makeContext(
}
describe("PromptBuilder", () => {
afterEach(() => {
delete process.env["AGENT_SKILL_MD_MODE"];
delete process.env["AGENT_USER_MEMORY_ENABLED"];
});
it("includes intent-matched examples in system prompt", () => {
const messages = PromptBuilder.forDerived(
makeDerived(),
@@ -85,4 +90,52 @@ describe("PromptBuilder", () => {
expect(messages[5]?.content).toBe("message-7");
expect(messages[0]?.content).toContain("이전 대화 요약");
});
it("uses SKILL.md L1/L2 prompt layers when enabled", () => {
const basePrompt = PromptBuilder.forDerived(
makeDerived(),
makeSession(),
makeContext(),
).buildSystemPrompt();
process.env["AGENT_SKILL_MD_MODE"] = "true";
const messages = PromptBuilder.forDerived(
makeDerived(),
makeSession(),
makeContext(),
).build();
expect(messages[0]?.content).toContain("[사용 가능한 SKILL.md]");
expect(messages[0]?.content).toContain("[ACTIVE_SKILL:onboarding]");
expect(messages[0]?.content).toContain("카카오톡 오픈채팅 새내기 등록");
expect(messages[0]?.content).toContain("[현재 유저 상태]");
expect(messages[0]?.content.length ?? 0).toBeLessThan(basePrompt.length * 0.65);
});
it("injects user memory and adaptive examples when enabled", () => {
process.env["AGENT_USER_MEMORY_ENABLED"] = "true";
const messages = PromptBuilder.forDerived(
makeDerived({
intent: { kind: "nickname-question" },
phase: { kind: "nickname-rejected", current: "bad", issues: [] },
}),
makeSession(),
makeContext({
userProfile: {
firstSeenAt: "2026-05-01T00:00:00.000Z",
flaggedIssues: [],
lastSeenAt: "2026-05-02T00:00:00.000Z",
pastIntents: [{ count: 3, kind: "nickname-question" }],
preferredStyle: "concise",
totalTurns: 3,
userId: "user_12345678",
},
}),
).build();
expect(messages[0]?.content).toContain("[유저 메모리]");
expect(messages[0]?.content).toContain("preferredStyle: concise");
expect(messages[0]?.content).toContain("닉네임이 왜 안 돼요?");
});
});
+135 -9
View File
@@ -1,9 +1,14 @@
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { SkillRegistry } from "@skills/skill-loader";
import type { ChatMessage } from "@llm/types";
import {
getAgentSkillMdMode,
getAgentUserMemoryEnabled,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
import type { OnboardingSession, PromptContext } from "../types.js";
import type { Example, OnboardingSession, PromptContext } from "../types.js";
import {
contextLayer,
examplesLayer,
@@ -12,6 +17,7 @@ import {
stateLayer,
} from "./layers.js";
import { buildToolPolicy } from "./tool-policy.js";
import { defaultExampleRetriever } from "./example-store.js";
export type PromptLayerName =
| "context"
@@ -35,11 +41,22 @@ export interface PromptVariantManifest {
diff: PromptLayerDiff[];
expectedImprovement?: string | undefined;
rationale?: string | undefined;
requiredResources?: string[] | undefined;
requiredTools?: string[] | undefined;
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
variantId: string;
}
const variantCache = new Map<string, PromptVariantManifest | null>();
function getSkillRootDir(): string {
return resolve(process.cwd(), "src", "skills");
}
function getOnboardingSkillRegistry(): SkillRegistry {
return SkillRegistry.load(getSkillRootDir());
}
function getLayerTarget(layer: PromptLayerName): Exclude<PromptLayerName, "tool-policy"> {
return layer === "tool-policy" ? "policy" : layer;
}
@@ -105,6 +122,76 @@ export function loadPromptVariantManifest(
}
}
function skillIndexLayer(registry: SkillRegistry): string {
const lines = registry.list().map(
(manifest) =>
`- ${manifest.name}: ${manifest.description} (version=${manifest.version}, trustTier=${manifest.trustTier})`,
);
return ["[사용 가능한 SKILL.md]", ...lines].join("\n");
}
function skillMdPolicyLayer(
derived: DerivedState,
toolPolicy?: string,
): string {
const lines = ["[의도별 원칙]"];
switch (derived.intent.kind) {
case "greeting":
lines.push("- 짧은 인사만 1문장.");
break;
case "acknowledgement":
lines.push("- 짧은 확인이나 마무리만 1문장.");
break;
case "status-question":
lines.push("- 완료된 것은 빼고 남은 절차만.");
break;
case "help-where":
lines.push("- 위치나 경로만.");
break;
case "nickname-question":
lines.push("- 닉네임 형식/오류 원인만.");
break;
case "photo-question":
lines.push("- 하트 인증에 대해서만.");
break;
case "discord-question":
case "discord-verify-question":
lines.push("- 디스코드 질문에만 답하고 확인 안 된 내용은 단정하지 마.");
break;
case "age-question":
lines.push("- 나이 제한만.");
break;
default:
lines.push("- 질문 범위를 넘지 마.");
break;
}
if (toolPolicy) {
lines.push(toolPolicy);
}
return lines.join("\n");
}
export function buildSkillResourceAddendum(resourceKeys: readonly string[]): string {
if (!getAgentSkillMdMode()) {
return "";
}
const registry = getOnboardingSkillRegistry();
const uniqueKeys = [...new Set(resourceKeys.map((key) => key.trim()).filter(Boolean))];
const resources = uniqueKeys.flatMap((key) => {
const body = registry.loadResourceSync("onboarding", key);
return body ? [`[RESOURCE:${key}]\n${body}`] : [];
});
return resources.length > 0
? ["[SKILL_RESOURCES]", ...resources].join("\n\n")
: "";
}
export class PromptBuilder {
private constructor(
private readonly derived: DerivedState,
@@ -139,18 +226,22 @@ export class PromptBuilder {
const promptContext: PromptContext = {
...this.context,
conversationHistory: promptHistory,
userProfile: getAgentUserMemoryEnabled()
? this.context.userProfile
: undefined,
};
const layers = {
persona: personaLayer(),
policy: policyLayer(this.derived.intent, {
toolPolicy:
this.derived.responseMode === "llm-with-tools"
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
: undefined,
}),
examples: getAgentSkillMdMode()
? ""
: examplesLayer(this.derived.intent, {
examples: this.buildExamples(),
}),
persona: getAgentSkillMdMode()
? this.buildSkillMdPersonaLayer()
: personaLayer(),
policy: this.buildPolicyLayer(),
state: stateLayer(this.session, this.derived),
examples: examplesLayer(this.derived.intent),
context: contextLayer(promptContext),
};
@@ -169,6 +260,41 @@ export class PromptBuilder {
.join("\n\n");
}
private buildSkillMdPersonaLayer(): string {
const registry = getOnboardingSkillRegistry();
const body = registry.loadBodySync("onboarding");
return [
skillIndexLayer(registry),
body ? `[ACTIVE_SKILL:onboarding]\n${body}` : "",
]
.filter((layer) => layer.trim().length > 0)
.join("\n\n");
}
private buildExamples(): readonly Example[] | undefined {
if (!getAgentUserMemoryEnabled() || !this.context.userProfile) {
return undefined;
}
return defaultExampleRetriever.retrieve({
intent: this.derived.intent,
phase: this.derived.phase,
preferredStyle: this.context.userProfile.preferredStyle,
});
}
private buildPolicyLayer(): string {
const toolPolicy =
this.derived.responseMode === "llm-with-tools"
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
: undefined;
return getAgentSkillMdMode()
? skillMdPolicyLayer(this.derived, toolPolicy)
: policyLayer(this.derived.intent, { toolPolicy });
}
build(): ChatMessage[] {
const promptHistory = this.context.summary
? this.context.conversationHistory.slice(-5)
@@ -0,0 +1,25 @@
import { describe, expect, it } from "vitest";
import { ExampleRetriever } from "./example-store.js";
describe("ExampleRetriever", () => {
it("keeps intent-matched examples first", () => {
const examples = new ExampleRetriever().retrieve({
intent: { kind: "age-question" },
phase: { kind: "fresh" },
preferredStyle: "neutral",
});
expect(examples[0]?.user).toBe("나이 제한 있나요?");
});
it("uses phase and style to rank fallback examples", () => {
const examples = new ExampleRetriever().retrieve({
intent: { kind: "unknown" },
phase: { kind: "photo-pending" },
preferredStyle: "warm",
});
expect(examples.some((example) => example.user.includes("하트"))).toBe(true);
});
});
@@ -0,0 +1,70 @@
import type { Example } from "../types.js";
import type { Intent, OnboardingPhase } from "../state/onboarding-state.js";
import type { PreferredStyle } from "../state/user-memory.js";
import { EXAMPLES_BY_INTENT } from "./examples.js";
export interface ExampleQuery {
intent: Intent;
phase: OnboardingPhase;
preferredStyle?: PreferredStyle | undefined;
}
function scoreStyle(example: Example, style: PreferredStyle | undefined): number {
if (!style) {
return 0;
}
const text = `${example.user}\n${example.assistant}`;
if (style === "concise") {
return example.assistant.length <= 45 ? 2 : 0;
}
if (style === "warm") {
return /[!~]|천천히|편하게|좋아요|어서오세요/u.test(text) ? 2 : 0;
}
return /[!~]/u.test(example.assistant) ? 0 : 1;
}
function scorePhase(example: Example, phase: OnboardingPhase): number {
const text = `${example.user}\n${example.assistant}`;
switch (phase.kind) {
case "nickname-pending":
case "nickname-rejected":
return /닉네임|티어|형식/u.test(text) ? 1 : 0;
case "photo-pending":
case "photo-rejected":
return /하트|인증|스크린샷/u.test(text) ? 1 : 0;
case "step1-completed":
case "completed":
return /디스코드|본인인증방|구인방/u.test(text) ? 1 : 0;
default:
return 0;
}
}
export class ExampleRetriever {
retrieve(query: ExampleQuery, limit = 3): Example[] {
const primary = EXAMPLES_BY_INTENT[query.intent.kind].map((example) => ({
example,
score: 10 + scorePhase(example, query.phase) + scoreStyle(example, query.preferredStyle),
}));
const fallback = EXAMPLES_BY_INTENT["help-how"].map((example) => ({
example,
score: scorePhase(example, query.phase) + scoreStyle(example, query.preferredStyle),
}));
return [...primary, ...fallback]
.sort((left, right) => right.score - left.score)
.map((item) => item.example)
.filter((example, index, examples) =>
examples.findIndex(
(candidate) =>
candidate.user === example.user &&
candidate.assistant === example.assistant,
) === index,
)
.slice(0, limit);
}
}
export const defaultExampleRetriever = new ExampleRetriever();
+43 -2
View File
@@ -2,6 +2,7 @@ import type { ConversationMessage } from '@agent/types';
import type { RoomMessage } from '@room/types';
import type { DerivedState, Intent } from '../state/onboarding-state.js';
import type { Blackboard, Example, OnboardingSession, PromptContext } from '../types.js';
import type { UserProfile } from '../state/user-memory.js';
import {
HEART_RULE_TEXT,
MIN_AGE,
@@ -101,6 +102,32 @@ function formatExample(example: Example): string {
return `유저: ${example.user}\n응답: ${example.assistant}`;
}
function formatUserProfile(profile: UserProfile | null | undefined): string {
if (!profile) {
return '(없음)';
}
const intents =
profile.pastIntents.length > 0
? profile.pastIntents
.slice(0, 5)
.map((intent) => `${intent.kind}:${intent.count}`)
.join(', ')
: '(없음)';
const issues =
profile.flaggedIssues.length > 0
? profile.flaggedIssues.slice(-3).join(', ')
: '(없음)';
return [
`- totalTurns: ${profile.totalTurns}`,
`- preferredStyle: ${profile.preferredStyle}`,
`- pastIntents: ${intents}`,
`- flaggedIssues: ${issues}`,
`- completedOnboardingAt: ${profile.completedOnboardingAt ?? '(없음)'}`,
].join('\n');
}
export function personaLayer(): string {
return `너는 롤림푸스(lolympus) 카카오톡 오픈채팅에서 새내기 등록을 도와주는 친절한 선배야.
처음 온 멤버가 편안하게 등록을 마칠 수 있도록 자연스럽고 정확하게 안내해줘.
@@ -208,8 +235,15 @@ export function stateLayer(session: OnboardingSession | null, derived: DerivedSt
].join('\n');
}
export function examplesLayer(intent: Intent, count = 3): string {
const examples = EXAMPLES_BY_INTENT[intent.kind].slice(0, count);
export function examplesLayer(
intent: Intent,
options?: {
count?: number | undefined;
examples?: readonly Example[] | undefined;
},
): string {
const count = options?.count ?? 3;
const examples = options?.examples ?? EXAMPLES_BY_INTENT[intent.kind].slice(0, count);
const rendered = examples.length > 0 ? examples.map(formatExample).join('\n\n') : '(예시 없음)';
return `[응답 예시]\n${rendered}`;
@@ -228,5 +262,12 @@ export function contextLayer(context: PromptContext): string {
'',
'[메모]',
formatBlackboard(context.blackboard),
...(context.userProfile
? [
'',
'[유저 메모리]',
formatUserProfile(context.userProfile),
]
: []),
].join('\n');
}