Add skill governance and runtime routing
This commit is contained in:
1 parent
6f09408423
commit
41dcfcdef7
71 files changed
+7057
-102
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { PromptBuilder } from "./builder.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
@@ -60,6 +60,11 @@ function makeContext(
|
||||
}
|
||||
|
||||
describe("PromptBuilder", () => {
|
||||
afterEach(() => {
|
||||
delete process.env["AGENT_SKILL_MD_MODE"];
|
||||
delete process.env["AGENT_USER_MEMORY_ENABLED"];
|
||||
});
|
||||
|
||||
it("includes intent-matched examples in system prompt", () => {
|
||||
const messages = PromptBuilder.forDerived(
|
||||
makeDerived(),
|
||||
@@ -85,4 +90,52 @@ describe("PromptBuilder", () => {
|
||||
expect(messages[5]?.content).toBe("message-7");
|
||||
expect(messages[0]?.content).toContain("이전 대화 요약");
|
||||
});
|
||||
|
||||
it("uses SKILL.md L1/L2 prompt layers when enabled", () => {
|
||||
const basePrompt = PromptBuilder.forDerived(
|
||||
makeDerived(),
|
||||
makeSession(),
|
||||
makeContext(),
|
||||
).buildSystemPrompt();
|
||||
process.env["AGENT_SKILL_MD_MODE"] = "true";
|
||||
|
||||
const messages = PromptBuilder.forDerived(
|
||||
makeDerived(),
|
||||
makeSession(),
|
||||
makeContext(),
|
||||
).build();
|
||||
|
||||
expect(messages[0]?.content).toContain("[사용 가능한 SKILL.md]");
|
||||
expect(messages[0]?.content).toContain("[ACTIVE_SKILL:onboarding]");
|
||||
expect(messages[0]?.content).toContain("카카오톡 오픈채팅 새내기 등록");
|
||||
expect(messages[0]?.content).toContain("[현재 유저 상태]");
|
||||
expect(messages[0]?.content.length ?? 0).toBeLessThan(basePrompt.length * 0.65);
|
||||
});
|
||||
|
||||
it("injects user memory and adaptive examples when enabled", () => {
|
||||
process.env["AGENT_USER_MEMORY_ENABLED"] = "true";
|
||||
|
||||
const messages = PromptBuilder.forDerived(
|
||||
makeDerived({
|
||||
intent: { kind: "nickname-question" },
|
||||
phase: { kind: "nickname-rejected", current: "bad", issues: [] },
|
||||
}),
|
||||
makeSession(),
|
||||
makeContext({
|
||||
userProfile: {
|
||||
firstSeenAt: "2026-05-01T00:00:00.000Z",
|
||||
flaggedIssues: [],
|
||||
lastSeenAt: "2026-05-02T00:00:00.000Z",
|
||||
pastIntents: [{ count: 3, kind: "nickname-question" }],
|
||||
preferredStyle: "concise",
|
||||
totalTurns: 3,
|
||||
userId: "user_12345678",
|
||||
},
|
||||
}),
|
||||
).build();
|
||||
|
||||
expect(messages[0]?.content).toContain("[유저 메모리]");
|
||||
expect(messages[0]?.content).toContain("preferredStyle: concise");
|
||||
expect(messages[0]?.content).toContain("닉네임이 왜 안 돼요?");
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,14 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { SkillRegistry } from "@skills/skill-loader";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import {
|
||||
getAgentSkillMdMode,
|
||||
getAgentUserMemoryEnabled,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession, PromptContext } from "../types.js";
|
||||
import type { Example, OnboardingSession, PromptContext } from "../types.js";
|
||||
import {
|
||||
contextLayer,
|
||||
examplesLayer,
|
||||
@@ -12,6 +17,7 @@ import {
|
||||
stateLayer,
|
||||
} from "./layers.js";
|
||||
import { buildToolPolicy } from "./tool-policy.js";
|
||||
import { defaultExampleRetriever } from "./example-store.js";
|
||||
|
||||
export type PromptLayerName =
|
||||
| "context"
|
||||
@@ -35,11 +41,22 @@ export interface PromptVariantManifest {
|
||||
diff: PromptLayerDiff[];
|
||||
expectedImprovement?: string | undefined;
|
||||
rationale?: string | undefined;
|
||||
requiredResources?: string[] | undefined;
|
||||
requiredTools?: string[] | undefined;
|
||||
trustTier?: "T1" | "T2" | "T3" | "T4" | undefined;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||
|
||||
function getSkillRootDir(): string {
|
||||
return resolve(process.cwd(), "src", "skills");
|
||||
}
|
||||
|
||||
function getOnboardingSkillRegistry(): SkillRegistry {
|
||||
return SkillRegistry.load(getSkillRootDir());
|
||||
}
|
||||
|
||||
function getLayerTarget(layer: PromptLayerName): Exclude<PromptLayerName, "tool-policy"> {
|
||||
return layer === "tool-policy" ? "policy" : layer;
|
||||
}
|
||||
@@ -105,6 +122,76 @@ export function loadPromptVariantManifest(
|
||||
}
|
||||
}
|
||||
|
||||
function skillIndexLayer(registry: SkillRegistry): string {
|
||||
const lines = registry.list().map(
|
||||
(manifest) =>
|
||||
`- ${manifest.name}: ${manifest.description} (version=${manifest.version}, trustTier=${manifest.trustTier})`,
|
||||
);
|
||||
|
||||
return ["[사용 가능한 SKILL.md]", ...lines].join("\n");
|
||||
}
|
||||
|
||||
function skillMdPolicyLayer(
|
||||
derived: DerivedState,
|
||||
toolPolicy?: string,
|
||||
): string {
|
||||
const lines = ["[의도별 원칙]"];
|
||||
|
||||
switch (derived.intent.kind) {
|
||||
case "greeting":
|
||||
lines.push("- 짧은 인사만 1문장.");
|
||||
break;
|
||||
case "acknowledgement":
|
||||
lines.push("- 짧은 확인이나 마무리만 1문장.");
|
||||
break;
|
||||
case "status-question":
|
||||
lines.push("- 완료된 것은 빼고 남은 절차만.");
|
||||
break;
|
||||
case "help-where":
|
||||
lines.push("- 위치나 경로만.");
|
||||
break;
|
||||
case "nickname-question":
|
||||
lines.push("- 닉네임 형식/오류 원인만.");
|
||||
break;
|
||||
case "photo-question":
|
||||
lines.push("- 하트 인증에 대해서만.");
|
||||
break;
|
||||
case "discord-question":
|
||||
case "discord-verify-question":
|
||||
lines.push("- 디스코드 질문에만 답하고 확인 안 된 내용은 단정하지 마.");
|
||||
break;
|
||||
case "age-question":
|
||||
lines.push("- 나이 제한만.");
|
||||
break;
|
||||
default:
|
||||
lines.push("- 질문 범위를 넘지 마.");
|
||||
break;
|
||||
}
|
||||
|
||||
if (toolPolicy) {
|
||||
lines.push(toolPolicy);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function buildSkillResourceAddendum(resourceKeys: readonly string[]): string {
|
||||
if (!getAgentSkillMdMode()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const registry = getOnboardingSkillRegistry();
|
||||
const uniqueKeys = [...new Set(resourceKeys.map((key) => key.trim()).filter(Boolean))];
|
||||
const resources = uniqueKeys.flatMap((key) => {
|
||||
const body = registry.loadResourceSync("onboarding", key);
|
||||
return body ? [`[RESOURCE:${key}]\n${body}`] : [];
|
||||
});
|
||||
|
||||
return resources.length > 0
|
||||
? ["[SKILL_RESOURCES]", ...resources].join("\n\n")
|
||||
: "";
|
||||
}
|
||||
|
||||
export class PromptBuilder {
|
||||
private constructor(
|
||||
private readonly derived: DerivedState,
|
||||
@@ -139,18 +226,22 @@ export class PromptBuilder {
|
||||
const promptContext: PromptContext = {
|
||||
...this.context,
|
||||
conversationHistory: promptHistory,
|
||||
userProfile: getAgentUserMemoryEnabled()
|
||||
? this.context.userProfile
|
||||
: undefined,
|
||||
};
|
||||
|
||||
const layers = {
|
||||
persona: personaLayer(),
|
||||
policy: policyLayer(this.derived.intent, {
|
||||
toolPolicy:
|
||||
this.derived.responseMode === "llm-with-tools"
|
||||
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
|
||||
: undefined,
|
||||
}),
|
||||
examples: getAgentSkillMdMode()
|
||||
? ""
|
||||
: examplesLayer(this.derived.intent, {
|
||||
examples: this.buildExamples(),
|
||||
}),
|
||||
persona: getAgentSkillMdMode()
|
||||
? this.buildSkillMdPersonaLayer()
|
||||
: personaLayer(),
|
||||
policy: this.buildPolicyLayer(),
|
||||
state: stateLayer(this.session, this.derived),
|
||||
examples: examplesLayer(this.derived.intent),
|
||||
context: contextLayer(promptContext),
|
||||
};
|
||||
|
||||
@@ -169,6 +260,41 @@ export class PromptBuilder {
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
private buildSkillMdPersonaLayer(): string {
|
||||
const registry = getOnboardingSkillRegistry();
|
||||
const body = registry.loadBodySync("onboarding");
|
||||
|
||||
return [
|
||||
skillIndexLayer(registry),
|
||||
body ? `[ACTIVE_SKILL:onboarding]\n${body}` : "",
|
||||
]
|
||||
.filter((layer) => layer.trim().length > 0)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
private buildExamples(): readonly Example[] | undefined {
|
||||
if (!getAgentUserMemoryEnabled() || !this.context.userProfile) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return defaultExampleRetriever.retrieve({
|
||||
intent: this.derived.intent,
|
||||
phase: this.derived.phase,
|
||||
preferredStyle: this.context.userProfile.preferredStyle,
|
||||
});
|
||||
}
|
||||
|
||||
private buildPolicyLayer(): string {
|
||||
const toolPolicy =
|
||||
this.derived.responseMode === "llm-with-tools"
|
||||
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
|
||||
: undefined;
|
||||
|
||||
return getAgentSkillMdMode()
|
||||
? skillMdPolicyLayer(this.derived, toolPolicy)
|
||||
: policyLayer(this.derived.intent, { toolPolicy });
|
||||
}
|
||||
|
||||
build(): ChatMessage[] {
|
||||
const promptHistory = this.context.summary
|
||||
? this.context.conversationHistory.slice(-5)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExampleRetriever } from "./example-store.js";
|
||||
|
||||
describe("ExampleRetriever", () => {
|
||||
it("keeps intent-matched examples first", () => {
|
||||
const examples = new ExampleRetriever().retrieve({
|
||||
intent: { kind: "age-question" },
|
||||
phase: { kind: "fresh" },
|
||||
preferredStyle: "neutral",
|
||||
});
|
||||
|
||||
expect(examples[0]?.user).toBe("나이 제한 있나요?");
|
||||
});
|
||||
|
||||
it("uses phase and style to rank fallback examples", () => {
|
||||
const examples = new ExampleRetriever().retrieve({
|
||||
intent: { kind: "unknown" },
|
||||
phase: { kind: "photo-pending" },
|
||||
preferredStyle: "warm",
|
||||
});
|
||||
|
||||
expect(examples.some((example) => example.user.includes("하트"))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import type { Example } from "../types.js";
|
||||
import type { Intent, OnboardingPhase } from "../state/onboarding-state.js";
|
||||
import type { PreferredStyle } from "../state/user-memory.js";
|
||||
import { EXAMPLES_BY_INTENT } from "./examples.js";
|
||||
|
||||
export interface ExampleQuery {
|
||||
intent: Intent;
|
||||
phase: OnboardingPhase;
|
||||
preferredStyle?: PreferredStyle | undefined;
|
||||
}
|
||||
|
||||
function scoreStyle(example: Example, style: PreferredStyle | undefined): number {
|
||||
if (!style) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const text = `${example.user}\n${example.assistant}`;
|
||||
if (style === "concise") {
|
||||
return example.assistant.length <= 45 ? 2 : 0;
|
||||
}
|
||||
if (style === "warm") {
|
||||
return /[!~]|천천히|편하게|좋아요|어서오세요/u.test(text) ? 2 : 0;
|
||||
}
|
||||
|
||||
return /[!~]/u.test(example.assistant) ? 0 : 1;
|
||||
}
|
||||
|
||||
function scorePhase(example: Example, phase: OnboardingPhase): number {
|
||||
const text = `${example.user}\n${example.assistant}`;
|
||||
switch (phase.kind) {
|
||||
case "nickname-pending":
|
||||
case "nickname-rejected":
|
||||
return /닉네임|티어|형식/u.test(text) ? 1 : 0;
|
||||
case "photo-pending":
|
||||
case "photo-rejected":
|
||||
return /하트|인증|스크린샷/u.test(text) ? 1 : 0;
|
||||
case "step1-completed":
|
||||
case "completed":
|
||||
return /디스코드|본인인증방|구인방/u.test(text) ? 1 : 0;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export class ExampleRetriever {
|
||||
retrieve(query: ExampleQuery, limit = 3): Example[] {
|
||||
const primary = EXAMPLES_BY_INTENT[query.intent.kind].map((example) => ({
|
||||
example,
|
||||
score: 10 + scorePhase(example, query.phase) + scoreStyle(example, query.preferredStyle),
|
||||
}));
|
||||
const fallback = EXAMPLES_BY_INTENT["help-how"].map((example) => ({
|
||||
example,
|
||||
score: scorePhase(example, query.phase) + scoreStyle(example, query.preferredStyle),
|
||||
}));
|
||||
|
||||
return [...primary, ...fallback]
|
||||
.sort((left, right) => right.score - left.score)
|
||||
.map((item) => item.example)
|
||||
.filter((example, index, examples) =>
|
||||
examples.findIndex(
|
||||
(candidate) =>
|
||||
candidate.user === example.user &&
|
||||
candidate.assistant === example.assistant,
|
||||
) === index,
|
||||
)
|
||||
.slice(0, limit);
|
||||
}
|
||||
}
|
||||
|
||||
export const defaultExampleRetriever = new ExampleRetriever();
|
||||
@@ -2,6 +2,7 @@ import type { ConversationMessage } from '@agent/types';
|
||||
import type { RoomMessage } from '@room/types';
|
||||
import type { DerivedState, Intent } from '../state/onboarding-state.js';
|
||||
import type { Blackboard, Example, OnboardingSession, PromptContext } from '../types.js';
|
||||
import type { UserProfile } from '../state/user-memory.js';
|
||||
import {
|
||||
HEART_RULE_TEXT,
|
||||
MIN_AGE,
|
||||
@@ -101,6 +102,32 @@ function formatExample(example: Example): string {
|
||||
return `유저: ${example.user}\n응답: ${example.assistant}`;
|
||||
}
|
||||
|
||||
function formatUserProfile(profile: UserProfile | null | undefined): string {
|
||||
if (!profile) {
|
||||
return '(없음)';
|
||||
}
|
||||
|
||||
const intents =
|
||||
profile.pastIntents.length > 0
|
||||
? profile.pastIntents
|
||||
.slice(0, 5)
|
||||
.map((intent) => `${intent.kind}:${intent.count}`)
|
||||
.join(', ')
|
||||
: '(없음)';
|
||||
const issues =
|
||||
profile.flaggedIssues.length > 0
|
||||
? profile.flaggedIssues.slice(-3).join(', ')
|
||||
: '(없음)';
|
||||
|
||||
return [
|
||||
`- totalTurns: ${profile.totalTurns}`,
|
||||
`- preferredStyle: ${profile.preferredStyle}`,
|
||||
`- pastIntents: ${intents}`,
|
||||
`- flaggedIssues: ${issues}`,
|
||||
`- completedOnboardingAt: ${profile.completedOnboardingAt ?? '(없음)'}`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
export function personaLayer(): string {
|
||||
return `너는 롤림푸스(lolympus) 카카오톡 오픈채팅에서 새내기 등록을 도와주는 친절한 선배야.
|
||||
처음 온 멤버가 편안하게 등록을 마칠 수 있도록 자연스럽고 정확하게 안내해줘.
|
||||
@@ -208,8 +235,15 @@ export function stateLayer(session: OnboardingSession | null, derived: DerivedSt
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
export function examplesLayer(intent: Intent, count = 3): string {
|
||||
const examples = EXAMPLES_BY_INTENT[intent.kind].slice(0, count);
|
||||
export function examplesLayer(
|
||||
intent: Intent,
|
||||
options?: {
|
||||
count?: number | undefined;
|
||||
examples?: readonly Example[] | undefined;
|
||||
},
|
||||
): string {
|
||||
const count = options?.count ?? 3;
|
||||
const examples = options?.examples ?? EXAMPLES_BY_INTENT[intent.kind].slice(0, count);
|
||||
const rendered = examples.length > 0 ? examples.map(formatExample).join('\n\n') : '(예시 없음)';
|
||||
|
||||
return `[응답 예시]\n${rendered}`;
|
||||
@@ -228,5 +262,12 @@ export function contextLayer(context: PromptContext): string {
|
||||
'',
|
||||
'[메모]',
|
||||
formatBlackboard(context.blackboard),
|
||||
...(context.userProfile
|
||||
? [
|
||||
'',
|
||||
'[유저 메모리]',
|
||||
formatUserProfile(context.userProfile),
|
||||
]
|
||||
: []),
|
||||
].join('\n');
|
||||
}
|
||||
Reference in new issue
Block a user