Add bounded skill handoff graph

This commit is contained in:
dahyeon committed 2026-05-18 17:22:12 -07:00
1 parent 54bc295ead
commit c14adac162
42 files changed
+2243 -31

No files matched your search

+62
View File
@@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import { parseControlTokens } from "./control-token-parser.js";
describe("parseControlTokens", () => {
it("extracts handoff control tokens and strips them from text", () => {
const parsed = parseControlTokens(
"잠시 확인할게요. [HANDOFF:moderation:policy-risk]",
);
expect(parsed.stripped).toBe("잠시 확인할게요.");
expect(parsed.tokens).toEqual([
{
kind: "handoff",
raw: "[HANDOFF:moderation:policy-risk]",
reason: "policy-risk",
targetAgent: "moderation",
},
]);
});
it("extracts no-response tokens", () => {
const parsed = parseControlTokens("[NO_RESPONSE]");
expect(parsed.stripped).toBe("");
expect(parsed.tokens).toEqual([
{
kind: "no-response",
raw: "[NO_RESPONSE]",
},
]);
});
it("keeps invalid handoff reasons visible", () => {
const parsed = parseControlTokens("[HANDOFF:moderation:not-a-reason]");
expect(parsed.stripped).toBe("[HANDOFF:moderation:not-a-reason]");
expect(parsed.tokens).toEqual([]);
});
it("accepts all handoff reason codes", () => {
const reasons = [
"out-of-scope",
"missing-tool",
"low-confidence",
"validation-failed",
"same-agent-revisit",
"max-hop",
"high-risk-action",
"policy-risk",
];
for (const reason of reasons) {
const parsed = parseControlTokens(`[HANDOFF:supervisor:${reason}]`);
expect(parsed.tokens[0]).toMatchObject({
kind: "handoff",
reason,
targetAgent: "supervisor",
});
}
});
});
+55
View File
@@ -0,0 +1,55 @@
import type { HandoffReason } from "@agent/types";
const HANDOFF_REASONS = new Set<HandoffReason>([
"out-of-scope",
"missing-tool",
"low-confidence",
"validation-failed",
"same-agent-revisit",
"max-hop",
"high-risk-action",
"policy-risk",
]);
export type ControlToken =
| { kind: "no-response"; raw: string }
| {
kind: "handoff";
raw: string;
reason: HandoffReason;
targetAgent: string;
};
export function parseControlTokens(text: string): {
stripped: string;
tokens: ControlToken[];
} {
const tokens: ControlToken[] = [];
let stripped = text;
stripped = stripped.replace(/\[NO_RESPONSE\]/gu, (raw) => {
tokens.push({ kind: "no-response", raw });
return "";
});
stripped = stripped.replace(
/\[HANDOFF:([a-z0-9_-]+):([a-z-]+)\]/giu,
(raw, target: string, reason: string) => {
if (!HANDOFF_REASONS.has(reason as HandoffReason)) {
return raw;
}
tokens.push({
kind: "handoff",
raw,
reason: reason as HandoffReason,
targetAgent: target,
});
return "";
},
);
return {
stripped: stripped.trim(),
tokens,
};
}
+95
View File
@@ -0,0 +1,95 @@
import { describe, expect, it } from "vitest";
import type { HandoffReason } from "@agent/types";
import { decide } from "./trigger.js";
describe("escalation trigger", () => {
it("turns explicit handoff reasons into structured requests", () => {
const reasons: HandoffReason[] = [
"out-of-scope",
"missing-tool",
"low-confidence",
"validation-failed",
"same-agent-revisit",
"max-hop",
"high-risk-action",
"policy-risk",
];
for (const reason of reasons) {
const decision = decide({
classified: { confidence: 0.8 },
reason,
targetAgent: "supervisor",
taskSpec: "review",
});
expect(decision).toMatchObject({
reason,
shouldHandoff: true,
handoff: {
reason,
targetAgent: "supervisor",
taskSpec: "review",
},
});
}
});
it("escalates exhausted budgets and validation rejects", () => {
expect(
decide({
budget: {
estimatedCostUsd: 0,
handoffs: 4,
maxCostUsd: 0.5,
maxHandoffs: 4,
maxSameAgentVisits: 2,
maxToolCalls: 20,
toolCalls: 0,
visits: {},
},
}).handoff?.reason,
).toBe("max-hop");
expect(decide({ validation: { status: "reject" } }).handoff?.reason).toBe(
"validation-failed",
);
});
it("escalates low-confidence and uncertain high-risk responses", () => {
expect(
decide({ classified: { confidence: 0.2 } }).handoff?.reason,
).toBe("low-confidence");
expect(
decide({
capabilityCard: {
canHandle: ["policy"],
cannotHandle: [],
domain: "moderation",
handoffTargets: ["supervisor"],
outputContract: { format: "text", maxChars: "500" },
riskTier: "high",
},
classified: { confidence: 0.6 },
}).handoff?.reason,
).toBe("high-risk-action");
});
it("does not escalate ordinary confident responses", () => {
expect(
decide({
capabilityCard: {
canHandle: ["help"],
cannotHandle: [],
domain: "onboarding",
handoffTargets: ["supervisor"],
outputContract: { format: "text", maxChars: "500" },
riskTier: "medium",
},
classified: { confidence: 0.9 },
}),
).toEqual({ shouldHandoff: false });
});
});
+56
View File
@@ -0,0 +1,56 @@
import type { HandoffReason, HandoffRequest } from "@agent/types";
import type { BudgetSnapshot } from "@agent/budget";
import type { SkillCapabilityCard } from "@skills/skill-loader";
export interface EscalationDecision {
handoff?: HandoffRequest | undefined;
reason?: HandoffReason | undefined;
shouldHandoff: boolean;
}
export function decide(input: {
budget?: BudgetSnapshot | undefined;
capabilityCard?: SkillCapabilityCard | undefined;
classified?: { confidence?: number | undefined } | undefined;
reason?: HandoffReason | undefined;
targetAgent?: string | undefined;
taskSpec?: string | undefined;
validation?: { status?: string | undefined } | undefined;
}): EscalationDecision {
const confidence = input.classified?.confidence;
let reason: HandoffReason | undefined = input.reason;
if (!reason) {
if (input.budget && input.budget.handoffs >= input.budget.maxHandoffs) {
reason = "max-hop";
} else if (input.validation?.status === "reject") {
reason = "validation-failed";
} else if (typeof confidence === "number" && confidence < 0.35) {
reason = "low-confidence";
} else if (
input.capabilityCard?.riskTier === "high" &&
typeof confidence === "number" &&
confidence < 0.7
) {
reason = "high-risk-action";
}
}
if (!reason) {
return { shouldHandoff: false };
}
return {
handoff: {
confidence: confidence ?? 0.5,
evidenceRefs: [],
expectedOutput: "text",
reason,
risk: input.capabilityCard?.riskTier ?? "medium",
targetAgent: input.targetAgent ?? "supervisor",
taskSpec: input.taskSpec ?? "Escalate this request to the appropriate skill.",
},
reason,
shouldHandoff: true,
};
}
+9
View File
@@ -3,6 +3,15 @@ name: moderation
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
version: 1
priority: 50
domain: moderation
canHandle: [rules, violation-report, appeal, sanctions, policy-risk]
cannotHandle: [onboarding-status, nickname-format, photo-verification, discord-onboarding]
handoffTargets: [onboarding, supervisor]
riskTier: high
outputContract:
format: text
maxChars: "500"
requiredFields: text
intents: [greeting, rule-question, violation-report, appeal, off-topic]
triggers: [user-message]
tools: [lookup-rule]
+23
View File
@@ -6,6 +6,7 @@ import type {
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader";
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
@@ -22,6 +23,24 @@ type RuleTopic =
| "sanctions"
| "spam";
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: ["rules", "violation-report", "appeal", "sanctions", "policy-risk"],
cannotHandle: [
"onboarding-status",
"nickname-format",
"photo-verification",
"discord-onboarding",
],
domain: "moderation",
handoffTargets: ["onboarding", "supervisor"],
outputContract: {
format: "text",
maxChars: "500",
requiredFields: "text",
},
riskTier: "high",
};
function formatHistory(history: ConversationMessage[]): string {
if (history.length === 0) {
return "(none)";
@@ -156,6 +175,10 @@ export class ModerationSkill implements Skill {
return request.skill === this.name;
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
conversationHistory: ConversationMessage[],
+9
View File
@@ -3,6 +3,15 @@ name: onboarding
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
version: 1
priority: 70
domain: onboarding
canHandle: [new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed, onboarding-help]
cannotHandle: [moderation-report, sanctions, appeal, policy-enforcement]
handoffTargets: [moderation, supervisor]
riskTier: medium
outputContract:
format: text
maxChars: "500"
requiredFields: text
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
@@ -79,4 +79,14 @@ describe("validateOnboardingResponse", () => {
expect(result.status).toBe("pass");
});
it("passes handoff control tokens", () => {
const result = validateOnboardingResponse(
"[HANDOFF:moderation:policy-risk]",
makeDerived(),
);
expect(result.status).toBe("pass");
expect(result.reasons).toContain("handoff");
});
});
+16 -6
View File
@@ -1,5 +1,6 @@
import { trace } from "@opentelemetry/api";
import { getMaxResponseLength } from "../../../config.js";
import { parseControlTokens } from "../../control-token-parser.js";
import { VALID_TIER_SET } from "../constants.js";
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
import type { DerivedState } from "../state/onboarding-state.js";
@@ -131,15 +132,12 @@ export function validateOnboardingResponse(
derived: DerivedState,
options?: { allowShortResponse?: boolean | undefined },
): ValidationResult {
const text = postProcessResponse(draft);
const parsedTokens = parseControlTokens(postProcessResponse(draft));
const text = parsedTokens.stripped;
const rejectReasons: string[] = [];
const softWarnReasons: string[] = [];
if (!text) {
rejectReasons.push("empty");
}
if (text.includes("[NO_RESPONSE]")) {
if (parsedTokens.tokens.some((token) => token.kind === "no-response")) {
return markValidation({
status: "pass",
text: "[NO_RESPONSE]",
@@ -147,6 +145,18 @@ export function validateOnboardingResponse(
});
}
if (parsedTokens.tokens.some((token) => token.kind === "handoff")) {
return markValidation({
status: "pass",
text,
reasons: ["handoff"],
});
}
if (!text) {
rejectReasons.push("empty");
}
if (!options?.allowShortResponse && text.length < 10) {
rejectReasons.push("minLength");
}
+31
View File
@@ -5,6 +5,7 @@ import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import type { SkillCapabilityCard } from '@skills/skill-loader';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { createOnboardingToolRegistry } from './tools.js';
@@ -68,6 +69,32 @@ import type {
StoredOnboardingSession,
} from './types.js';
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: [
'new-member',
'nickname-invalid',
'photo-failed',
'progress-update',
'timeout-warning',
'completed',
'onboarding-help',
],
cannotHandle: [
'moderation-report',
'sanctions',
'appeal',
'policy-enforcement',
],
domain: 'onboarding',
handoffTargets: ['moderation', 'supervisor'],
outputContract: {
format: 'text',
maxChars: '500',
requiredFields: 'text',
},
riskTier: 'medium',
};
export class OnboardingSkill implements Skill {
readonly name = 'onboarding';
readonly priority = 70;
@@ -99,6 +126,10 @@ export class OnboardingSkill implements Skill {
return request.skill === 'onboarding';
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
conversationHistory: ConversationMessage[],
+19
View File
@@ -21,6 +21,15 @@ function makeTempSkillRoot(): string {
"priority: 65",
"intents: [hello, help]",
"triggers: [user-message]",
"domain: demo-domain",
"canHandle: [demo-task]",
"cannotHandle: [other-task]",
"handoffTargets: [supervisor]",
"riskTier: medium",
"outputContract:",
" format: text",
" maxChars: \"500\"",
" requiredFields: text",
"tools: [demo-tool]",
"resources:",
" guide: ./resources/guide.md",
@@ -41,8 +50,18 @@ describe("SkillRegistry", () => {
expect(manifest).toMatchObject({
name: "demo",
description: "Demo skill.",
canHandle: ["demo-task"],
cannotHandle: ["other-task"],
domain: "demo-domain",
handoffTargets: ["supervisor"],
intents: ["hello", "help"],
outputContract: {
format: "text",
maxChars: "500",
requiredFields: "text",
},
priority: 65,
riskTier: "medium",
tools: ["demo-tool"],
trustTier: "T2",
version: 1,
+55 -1
View File
@@ -3,14 +3,36 @@ import { dirname, resolve } from "node:path";
import { getLogger } from "@obs/logger";
export type TrustTier = "T1" | "T2" | "T3" | "T4";
export type SkillRiskTier = "low" | "medium" | "high";
export interface SkillOutputContract {
format?: string | undefined;
maxChars?: string | undefined;
requiredFields?: string | undefined;
}
export interface SkillCapabilityCard {
canHandle: string[];
cannotHandle: string[];
domain: string;
handoffTargets: string[];
outputContract: SkillOutputContract;
riskTier: SkillRiskTier;
}
export interface SkillManifest {
bodyPath: string;
canHandle: string[];
cannotHandle: string[];
description: string;
domain: string;
handoffTargets: string[];
intents: string[];
name: string;
outputContract: SkillOutputContract;
priority: number;
resources: Record<string, string>;
riskTier: SkillRiskTier;
tools: string[];
triggers: string[];
trustTier: TrustTier;
@@ -121,10 +143,32 @@ function readResources(
);
}
function readStringMap(
fields: Record<string, unknown>,
key: string,
): Record<string, string> {
const raw = fields[key];
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
return {};
}
return Object.fromEntries(
Object.entries(raw as Record<string, unknown>).flatMap(([mapKey, value]) =>
typeof value === "string" && value.trim()
? [[mapKey, value.trim().replace(/^["']|["']$/gu, "")]]
: [],
),
);
}
function isTrustTier(value: unknown): value is TrustTier {
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
}
function isRiskTier(value: unknown): value is SkillRiskTier {
return value === "low" || value === "medium" || value === "high";
}
function isPriority(value: unknown): value is number {
return (
typeof value === "number" &&
@@ -147,6 +191,8 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
const priority = parsed.fields["priority"] ?? 50;
const version = parsed.fields["version"];
const trustTier = parsed.fields["trustTier"];
const domain = parsed.fields["domain"];
const riskTier = parsed.fields["riskTier"] ?? "low";
if (
typeof name !== "string" ||
@@ -155,20 +201,28 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
!description.trim() ||
!isPriority(priority) ||
typeof version !== "number" ||
!isTrustTier(trustTier)
!isTrustTier(trustTier) ||
!isRiskTier(riskTier)
) {
return null;
}
const outputContract = readStringMap(parsed.fields, "outputContract");
return {
body: parsed.body,
manifest: {
bodyPath: path,
canHandle: readStringArray(parsed.fields, "canHandle"),
cannotHandle: readStringArray(parsed.fields, "cannotHandle"),
description,
domain: typeof domain === "string" && domain.trim() ? domain : "general",
handoffTargets: readStringArray(parsed.fields, "handoffTargets"),
intents: readStringArray(parsed.fields, "intents"),
name,
outputContract,
priority,
resources: readResources(parsed.fields, skillDir),
riskTier,
tools: readStringArray(parsed.fields, "tools"),
triggers: readStringArray(parsed.fields, "triggers"),
trustTier,
+2
View File
@@ -3,10 +3,12 @@ import type {
AgentResponse,
ConversationMessage,
} from "@agent/types";
import type { SkillCapabilityCard } from "./skill-loader.js";
export interface Skill {
readonly name: string;
readonly priority: number;
capabilityCard?(): SkillCapabilityCard;
canHandle(request: AgentRequest): boolean;
handle(
request: AgentRequest,
+22
View File
@@ -0,0 +1,22 @@
---
name: supervisor
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
version: 1
priority: 10
domain: meta
canHandle: [*]
cannotHandle: []
handoffTargets: [onboarding, moderation]
riskTier: medium
outputContract:
format: text
maxChars: "500"
requiredFields: text
intents: []
triggers: [handoff, user-message]
tools: []
resources:
trustTier: T4
---
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
+9
View File
@@ -0,0 +1,9 @@
import type { Skill } from "@skills/skill";
import type { SkillDeps } from "@skills/skill-factory";
import { SupervisorSkill } from "./skill.js";
export { SupervisorSkill } from "./skill.js";
export function createSupervisorSkill(deps: SkillDeps): Skill {
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
}
@@ -0,0 +1,7 @@
export function getIntentKinds(): string[] {
return [];
}
export function getToolNames(): string[] {
return [];
}
+14
View File
@@ -0,0 +1,14 @@
import type { GarnetClient } from "@common/garnet";
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
export async function appendSupervisorReview(
garnetClient: Pick<GarnetClient, "appendStream">,
fields: Record<string, string>,
): Promise<void> {
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
});
}
+17
View File
@@ -0,0 +1,17 @@
import type { HandoffPacket } from "@agent/handoff-packet";
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
const text = [
packet?.userGoal,
packet?.summary,
packet?.handoffReason,
].join(" ");
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) {
return "moderation";
}
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) {
return "onboarding";
}
return null;
}
+90
View File
@@ -0,0 +1,90 @@
import { describe, expect, it } from "vitest";
import type { AlertPayload } from "@common/nats-subjects";
import { buildHandoffPacket } from "@agent/handoff-packet";
import type { AgentRequest, AgentResponse } from "@agent/types";
import { SupervisorSkill } from "./skill.js";
class FakeGarnet {
readonly streams: Array<{ key: string; fields: Record<string, unknown> }> = [];
appendStream(key: string, fields: Record<string, unknown>): Promise<string> {
this.streams.push({ key, fields });
return Promise.resolve("1-0");
}
setStringIfAbsent(): Promise<boolean> {
return Promise.resolve(true);
}
}
class FakeNats {
readonly published: AlertPayload[] = [];
publish<T = unknown>(_subject: string, payload: T): Promise<void> {
this.published.push(payload as AlertPayload);
return Promise.resolve();
}
}
const baseRequest: AgentRequest = {
chatroomId: "room-1",
nickname: "tester",
requestId: "request-1",
skill: "supervisor",
trigger: "handoff",
userId: "user-1",
};
const sourceResponse: AgentResponse = {
confidence: 0.3,
requestId: "request-1",
source: "template",
text: "needs review",
};
describe("SupervisorSkill", () => {
it("queues review and publishes alert for handoffs", async () => {
const garnet = new FakeGarnet();
const nats = new FakeNats();
const skill = new SupervisorSkill(garnet, nats);
const packet = buildHandoffPacket({
handoff: {
confidence: 0.3,
evidenceRefs: [],
expectedOutput: "text",
reason: "max-hop",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "가입 상태 확인",
},
request: baseRequest,
response: sourceResponse,
sourceChain: ["onboarding", "supervisor"],
});
const response = await skill.handle(
{ ...baseRequest, context: { handoffPacket: packet } },
[],
);
expect(response.text).toContain("운영자");
expect(response.metadata).toMatchObject({
supervisorReason: "max-hop",
suggestedTarget: "onboarding",
});
expect(garnet.streams[0]).toMatchObject({
key: "lolympus-agent:review-queue",
fields: {
reason: "max-hop",
requestId: "request-1",
target: "onboarding",
type: "supervisor-handoff",
},
});
expect(nats.published[0]).toMatchObject({
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
});
});
});
+93
View File
@@ -0,0 +1,93 @@
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
} from "@agent/types";
import { readHandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader";
import { appendSupervisorReview } from "./review-queue.js";
import { routeSupervisorTarget } from "./routing-rules.js";
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: ["*"],
cannotHandle: [],
domain: "meta",
handoffTargets: ["onboarding", "moderation"],
outputContract: {
format: "text",
maxChars: "500",
requiredFields: "text",
},
riskTier: "medium",
};
export class SupervisorSkill implements Skill {
readonly name = "supervisor";
readonly priority = 10;
private readonly alertService: AlertService;
constructor(
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
natsClient: Pick<NatsClient, "publish">,
) {
this.alertService = new AlertService(garnetClient, natsClient);
}
canHandle(request: AgentRequest): boolean {
return request.skill === this.name;
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
_conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const packet = readHandoffPacket(request);
const target = routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope";
await appendSupervisorReview(this.garnetClient, {
chatroomId: request.chatroomId,
reason,
requestId: request.requestId,
target: target ?? "human-review",
userId: request.userId,
});
metrics.humanOverrideTotal.inc({ kind: reason });
await this.alertService.sendAlert(
{
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
severity: "warn",
suggestedAction: "Review the supervisor handoff queue.",
threshold: 0,
timestamp: new Date().toISOString(),
value: 1,
window: "request",
},
{
dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`,
},
);
return {
confidence: 0.6,
metadata: {
supervisorReason: reason,
suggestedTarget: target,
},
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
};
}
}
+65
View File
@@ -0,0 +1,65 @@
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { SkillRegistry } from "./skill-loader.js";
import { TransitionMatrix } from "./transition-matrix.js";
function writeSkill(
root: string,
name: string,
handoffTargets: string[],
): void {
const skillDir = join(root, name);
mkdirSync(skillDir, { recursive: true });
writeFileSync(
join(skillDir, "SKILL.md"),
[
"---",
`name: ${name}`,
`description: ${name} skill.`,
"version: 1",
"priority: 50",
"domain: test",
"canHandle: [test]",
"cannotHandle: []",
`handoffTargets: [${handoffTargets.join(", ")}]`,
"riskTier: low",
"outputContract:",
" format: text",
" maxChars: \"500\"",
"trustTier: T2",
"---",
"",
"body",
].join("\n"),
);
}
describe("TransitionMatrix", () => {
it("allows declared handoffs and blocks self loops", () => {
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-"));
writeSkill(root, "onboarding", ["moderation"]);
writeSkill(root, "moderation", []);
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
expect(matrix.isAllowed("onboarding", "moderation")).toBe(true);
expect(matrix.isAllowed("onboarding", "supervisor")).toBe(true);
expect(matrix.isAllowed("onboarding", "onboarding")).toBe(false);
expect(matrix.isAllowed("moderation", "onboarding")).toBe(false);
});
it("reports unknown handoff targets", () => {
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-bad-"));
writeSkill(root, "onboarding", ["missing"]);
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
expect(matrix.validateGraph(["onboarding"])).toContain(
"onboarding: unknown handoff target missing",
);
});
});
+47
View File
@@ -0,0 +1,47 @@
import type { SkillRegistry } from "./skill-loader.js";
export class TransitionMatrix {
private constructor(private readonly transitions: Map<string, Set<string>>) {}
static fromRegistry(registry: SkillRegistry): TransitionMatrix {
const transitions = new Map<string, Set<string>>();
for (const manifest of registry.list()) {
const targets =
manifest.name === "supervisor"
? manifest.handoffTargets
: [...manifest.handoffTargets, "supervisor"];
transitions.set(
manifest.name,
new Set(targets),
);
}
return new TransitionMatrix(transitions);
}
isAllowed(from: string, to: string): boolean {
if (from === to) {
return false;
}
return this.transitions.get(from)?.has(to) ?? true;
}
targetsOf(from: string): string[] {
return [...(this.transitions.get(from) ?? new Set<string>())];
}
validateGraph(knownSkills?: readonly string[]): string[] {
const known = new Set([...(knownSkills ?? this.transitions.keys()), "supervisor"]);
const errors: string[] = [];
for (const [from, targets] of this.transitions.entries()) {
if (targets.has(from)) {
errors.push(`${from}: handoffTargets must not include itself`);
}
for (const target of targets) {
if (!known.has(target)) {
errors.push(`${from}: unknown handoff target ${target}`);
}
}
}
return errors;
}
}
+48 -1
View File
@@ -3,8 +3,10 @@ import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
import { TransitionMatrix } from "./transition-matrix.js";
import * as moderationBinding from "./moderation/manifest-binding.js";
import * as onboardingBinding from "./onboarding/manifest-binding.js";
import * as supervisorBinding from "./supervisor/manifest-binding.js";
interface ManifestBinding {
getIntentKinds(): string[];
@@ -14,6 +16,7 @@ interface ManifestBinding {
const BINDINGS: Record<string, ManifestBinding> = {
moderation: moderationBinding,
onboarding: onboardingBinding,
supervisor: supervisorBinding,
};
function sortUnique(values: readonly string[]): string[] {
@@ -32,12 +35,15 @@ function diff(left: readonly string[], right: readonly string[]): string[] {
function validateManifest(
manifest: SkillManifest,
binding: ManifestBinding | undefined,
knownSkills: readonly string[],
): string[] {
if (!binding) {
return [`${manifest.name}: missing manifest binding`];
}
const errors: string[] = [];
const capabilityErrors = validateCapabilityCard(manifest, knownSkills);
errors.push(...capabilityErrors);
const manifestIntents = sortUnique(manifest.intents);
const codeIntents = sortUnique(binding.getIntentKinds());
const unknownIntents = diff(manifestIntents, codeIntents);
@@ -66,6 +72,45 @@ function validateManifest(
return errors;
}
function validateCapabilityCard(
manifest: SkillManifest,
knownSkills: readonly string[],
): string[] {
const errors: string[] = [];
const canHandle = new Set(manifest.canHandle);
const overlap = manifest.cannotHandle.filter((item) => canHandle.has(item));
if (overlap.length > 0) {
errors.push(
`${manifest.name}: canHandle and cannotHandle overlap: ${format(overlap)}`,
);
}
const riskTier: string = manifest.riskTier;
if (riskTier !== "low" && riskTier !== "medium" && riskTier !== "high") {
errors.push(`${manifest.name}: invalid riskTier ${riskTier}`);
}
const knownTargets = new Set([...knownSkills, "supervisor"]);
const missingTargets = manifest.handoffTargets.filter(
(target) => !knownTargets.has(target),
);
if (missingTargets.length > 0) {
errors.push(
`${manifest.name}: unknown handoffTargets: ${format(missingTargets)}`,
);
}
if (manifest.handoffTargets.includes(manifest.name)) {
errors.push(`${manifest.name}: handoffTargets must not include itself`);
}
if (!manifest.domain.trim()) {
errors.push(`${manifest.name}: domain is required`);
}
if (!manifest.outputContract.format || !manifest.outputContract.maxChars) {
errors.push(
`${manifest.name}: outputContract.format and outputContract.maxChars are required`,
);
}
return errors;
}
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
const byPriority = new Map<number, string[]>();
for (const manifest of manifests) {
@@ -84,11 +129,13 @@ function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
const registry = SkillRegistry.load(rootDir);
const manifests = registry.list();
const knownSkills = manifests.map((manifest) => manifest.name);
return [
...manifests.flatMap((manifest) =>
validateManifest(manifest, BINDINGS[manifest.name]),
validateManifest(manifest, BINDINGS[manifest.name], knownSkills),
),
...validatePriorityConflicts(manifests),
...TransitionMatrix.fromRegistry(registry).validateGraph(knownSkills),
];
}