Add bounded skill handoff graph

This commit is contained in:
dahyeon committed 2026-05-18 17:22:12 -07:00
1 parent 54bc295ead
commit c14adac162
42 files changed
+2243 -31

No files matched your search

+249 -1
View File
@@ -1,7 +1,10 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
import { describe, expect, it } from "vitest";
import type { Skill } from "@skills/skill";
import { selectSkillForRequest } from "./agent.js";
import type { ConversationStore } from "@conv/conversation-store";
import type { NatsClient } from "@common/nats";
import type { TransitionMatrix } from "@skills/transition-matrix";
import { Agent, selectSkillForRequest } from "./agent.js";
function makeRequest(skill = "shared"): AgentRequest {
return {
@@ -32,6 +35,60 @@ function makeSkill(name: string, priority: number, canHandle = true): Skill {
};
}
class MemoryConversationStore {
readonly messages: ConversationMessage[] = [];
getHistory(): Promise<ConversationMessage[]> {
return Promise.resolve([...this.messages]);
}
appendMessage(
_chatroomId: string,
_userId: string,
message: ConversationMessage,
): Promise<void> {
this.messages.push(message);
return Promise.resolve();
}
}
class FakeNatsClient {
readonly responses: AgentResponse[] = [];
respond(_msg: unknown, response: AgentResponse): Promise<void> {
this.responses.push(response);
return Promise.resolve();
}
}
async function handleThroughAgent(
skills: readonly Skill[],
request: AgentRequest,
transitionMatrix?: TransitionMatrix,
): Promise<{
nats: FakeNatsClient;
store: MemoryConversationStore;
}> {
const nats = new FakeNatsClient();
const store = new MemoryConversationStore();
const agent = new Agent(
nats as unknown as NatsClient,
store as unknown as ConversationStore,
transitionMatrix,
);
for (const skill of skills) {
agent.registerSkill(skill);
}
await (agent as unknown as {
handleRequest(msg: { data: Uint8Array }): Promise<void>;
}).handleRequest({
data: new TextEncoder().encode(JSON.stringify(request)),
});
return { nats, store };
}
describe("selectSkillForRequest", () => {
it("selects the highest-priority matching skill", () => {
const request = makeRequest();
@@ -51,3 +108,194 @@ describe("selectSkillForRequest", () => {
expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull();
});
});
describe("Agent handoff dispatch", () => {
it("responds once with the final handoff target response", async () => {
const source: Skill = {
...makeSkill("source", 50),
handle: (request) =>
Promise.resolve({
confidence: 0.8,
metadata: {
handoffRequest: {
confidence: 0.8,
evidenceRefs: ["rule-1"],
expectedOutput: "text",
reason: "out-of-scope",
risk: "medium",
targetAgent: "target",
taskSpec: "continue",
},
},
requestId: request.requestId,
source: "template",
text: "handoff",
}),
};
const target: Skill = {
...makeSkill("target", 60),
canHandle: (request) => request.skill === "target",
handle: (request) =>
Promise.resolve({
confidence: 1,
requestId: request.requestId,
source: "template",
text: "final answer",
}),
};
const { nats, store } = await handleThroughAgent(
[source, target],
makeRequest("source"),
);
expect(nats.responses).toHaveLength(1);
expect(nats.responses[0]?.text).toBe("final answer");
expect(store.messages.map((message) => message.content)).toEqual([
"final answer",
]);
});
it("routes handoff control tokens through metadata", async () => {
const source: Skill = {
...makeSkill("source", 50),
handle: (request) =>
Promise.resolve({
confidence: 0.9,
requestId: request.requestId,
source: "template",
text: "[HANDOFF:target:missing-tool]",
}),
};
const target: Skill = {
...makeSkill("target", 60),
canHandle: (request) => request.skill === "target",
handle: (request) =>
Promise.resolve({
confidence: 1,
metadata: {
packetSeen: Boolean(request.context?.["handoffPacket"]),
},
requestId: request.requestId,
source: "template",
text: "handled elsewhere",
}),
};
const { nats } = await handleThroughAgent(
[source, target],
makeRequest("source"),
);
expect(nats.responses[0]?.text).toBe("handled elsewhere");
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
});
it("routes runtime verifier escalations to supervisor", async () => {
const previous = process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = "true";
try {
const moderation: Skill = {
...makeSkill("moderation", 50),
canHandle: (request) => request.skill === "moderation",
handle: (request) =>
Promise.resolve({
confidence: 0.9,
requestId: request.requestId,
source: "template",
text: "https://not-allowed.example",
}),
};
const supervisor: Skill = {
...makeSkill("supervisor", 90),
canHandle: (request) => request.skill === "supervisor",
handle: (request) =>
Promise.resolve({
confidence: 0.6,
metadata: {
packetSeen: Boolean(request.context?.["handoffPacket"]),
},
requestId: request.requestId,
source: "template",
text: "sent to review",
}),
};
const { nats } = await handleThroughAgent(
[moderation, supervisor],
makeRequest("moderation"),
);
expect(nats.responses[0]?.text).toBe("sent to review");
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
} finally {
if (previous === undefined) {
delete process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
} else {
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = previous;
}
}
});
it("reroutes illegal transitions to supervisor", async () => {
const source: Skill = {
...makeSkill("source", 50),
canHandle: (request) => request.skill === "source",
handle: (request) =>
Promise.resolve({
confidence: 0.9,
metadata: {
handoffRequest: {
confidence: 0.9,
evidenceRefs: [],
expectedOutput: "text",
reason: "out-of-scope",
risk: "medium",
targetAgent: "target",
taskSpec: "continue",
},
},
requestId: request.requestId,
source: "template",
text: "handoff",
}),
};
const target: Skill = {
...makeSkill("target", 60),
canHandle: (request) => request.skill === "target",
handle: (request) =>
Promise.resolve({
confidence: 1,
requestId: request.requestId,
source: "template",
text: "should not run",
}),
};
const supervisor: Skill = {
...makeSkill("supervisor", 10),
canHandle: (request) => request.skill === "supervisor",
handle: (request) =>
Promise.resolve({
confidence: 0.6,
metadata: {
packetSeen: Boolean(request.context?.["handoffPacket"]),
},
requestId: request.requestId,
source: "template",
text: "supervisor review",
}),
};
const matrix = {
isAllowed: () => false,
} as unknown as TransitionMatrix;
const { nats } = await handleThroughAgent(
[source, target, supervisor],
makeRequest("source"),
matrix,
);
expect(nats.responses[0]?.text).toBe("supervisor review");
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
});
});
+447 -19
View File
@@ -5,8 +5,22 @@ import { getRequestLogger, runWithContext } from "@obs/context";
import { getLogger } from "@obs/logger";
import { withSkillSpan, withSpan } from "@obs/spans";
import type { Skill } from "@skills/skill";
import type { AgentRequest, AgentResponse } from "./types.js";
import type { TransitionMatrix } from "@skills/transition-matrix";
import { parseControlTokens } from "@skills/control-token-parser";
import { decide as decideEscalation } from "@skills/escalation/trigger";
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
import { metrics } from "@obs/metrics";
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
import { buildHandoffPacket } from "./handoff-packet.js";
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
HandoffReason,
HandoffRequest,
} from "./types.js";
import {
getAgentRuntimeVerifierEnabled,
getAgentRequestTimeoutMs,
getRequestSubject,
getSafetyMarginMs,
@@ -43,6 +57,7 @@ export class Agent {
constructor(
private readonly natsClient: NatsClient,
private readonly conversationStore: ConversationStore,
private readonly transitionMatrix?: TransitionMatrix | undefined,
) {}
registerSkill(skill: Skill): void {
@@ -107,18 +122,28 @@ export class Agent {
"[Agent] Request received",
);
const skill = selectSkillForRequest(this.skills, request);
let skill = selectSkillForRequest(this.skills, request);
if (!skill) {
logger.info("[Agent] No skill found for request");
const response: AgentResponse = {
requestId: request.requestId,
text: "",
confidence: 0,
source: "template",
const supervisorRequest: AgentRequest = {
...request,
skill: "supervisor",
trigger: "handoff",
};
rootSpan.setAttribute("agent.source", "template");
await this.natsClient.respond(msg, response);
return;
skill = selectSkillForRequest(this.skills, supervisorRequest);
if (skill) {
request = supervisorRequest;
} else {
logger.info("[Agent] No skill found for request");
const response: AgentResponse = {
requestId: request.requestId,
text: "",
confidence: 0,
source: "template",
};
rootSpan.setAttribute("agent.source", "template");
await this.natsClient.respond(msg, response);
return;
}
}
try {
@@ -139,14 +164,17 @@ export class Agent {
);
}
const response = await withSkillSpan(
skill.name,
{
"agent.skill": skill.name,
"agent.trigger": request.trigger,
},
() => skill.handle(request, history),
);
const startedAt = Date.now();
const budget = new BudgetTracker();
budget.recordVisit(skill.name);
const response = await this.dispatch({
budget,
chain: [skill.name],
depth: 0,
history,
request,
skill,
});
if (response.text.trim()) {
await this.conversationStore.appendMessage(
@@ -161,6 +189,8 @@ export class Agent {
}
await this.natsClient.respond(msg, response);
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
rootSpan.setAttribute(
"agent.source",
response.text.trim() ? response.source : "empty",
@@ -200,4 +230,402 @@ export class Agent {
},
);
}
private async dispatch(input: {
budget: BudgetTracker;
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}): Promise<AgentResponse> {
const { budget, chain, depth, history, request, skill } = input;
const contextBase = {
chatroomId: request.chatroomId,
deadlineAt: new Date(
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
).toISOString(),
requestId: request.requestId,
skill: skill.name,
startedAt: new Date().toISOString(),
trigger: request.trigger,
userId: request.userId,
};
return runWithContext(
{
...contextBase,
budget: budget.snapshot(),
handoffChain: chain,
handoffDepth: depth,
},
async () => {
const response = await this.runSkillOnce({
chain,
depth,
history,
request,
skill,
});
const normalized = this.normalizeControlTokens(response, skill.name);
const verified = await this.applyRuntimeVerifier({
chain,
depth,
history,
request,
response: normalized,
skill,
});
const escalated = this.applyEscalationTrigger({
budget,
request,
response: verified,
skill,
});
const handoff = readHandoffRequest(escalated);
if (!handoff) {
return escalated;
}
const next = this.prepareHandoff({
budget,
chain,
currentSkill: skill.name,
depth,
handoff,
history,
request,
response: escalated,
});
if (!next) {
return this.makeSupervisorFallback(request, handoff.reason);
}
return this.dispatch(next);
},
);
}
private async runSkillOnce(input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}): Promise<AgentResponse> {
return withSkillSpan(
input.skill.name,
{
"agent.handoff_chain": input.chain.join(">"),
"agent.handoff_depth": input.depth,
"agent.skill": input.skill.name,
"agent.trigger": input.request.trigger,
},
() => input.skill.handle(input.request, input.history),
);
}
private normalizeControlTokens(
response: AgentResponse,
currentSkill: string,
): AgentResponse {
const parsed = parseControlTokens(response.text);
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
const metadata = { ...(response.metadata ?? {}) };
if (handoff?.kind === "handoff") {
metadata.handoffRequest = {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: handoff.reason,
risk: "medium",
targetAgent: handoff.targetAgent,
taskSpec: `Continue handling from ${currentSkill}`,
} satisfies HandoffRequest;
}
return {
...response,
metadata,
text: parsed.stripped,
};
}
private async applyRuntimeVerifier(input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
response: AgentResponse;
skill: Skill;
}): Promise<AgentResponse> {
if (!getAgentRuntimeVerifierEnabled()) {
return input.response;
}
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
if (!verifier) {
return input.response;
}
const verdict = await verifier({
request: input.request,
response: input.response,
skill: input.skill.name,
});
if (verdict === "pass") {
return input.response;
}
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
if (verdict === "retry" && input.depth === 0) {
const retryResponse = await this.runSkillOnce({
...input,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
runtimeVerifierRetry: true,
},
},
});
const normalizedRetry = this.normalizeControlTokens(
retryResponse,
input.skill.name,
);
if (readHandoffRequest(normalizedRetry)) {
return normalizedRetry;
}
const retryVerdict = await verifier({
request: input.request,
response: normalizedRetry,
skill: input.skill.name,
});
if (retryVerdict === "pass") {
return {
...normalizedRetry,
metadata: {
...(normalizedRetry.metadata ?? {}),
runtimeVerifier: "retry-pass",
},
};
}
metrics.validatorFailTotal.inc({
skill: input.skill.name,
stage: `retry-${retryVerdict}`,
});
return this.withRuntimeVerifierHandoff(normalizedRetry);
}
return this.withRuntimeVerifierHandoff(input.response);
}
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: "validation-failed",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "Runtime verifier rejected the response.",
} satisfies HandoffRequest,
},
};
}
private applyEscalationTrigger(input: {
budget: BudgetTracker;
request: AgentRequest;
response: AgentResponse;
skill: Skill;
}): AgentResponse {
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
return input.response;
}
const decision = decideEscalation({
budget: input.budget.snapshot(),
capabilityCard: input.skill.capabilityCard?.(),
classified: { confidence: input.response.confidence },
targetAgent: "supervisor",
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
});
if (!decision.handoff) {
return input.response;
}
return {
...input.response,
metadata: {
...(input.response.metadata ?? {}),
handoffRequest: decision.handoff,
},
};
}
private prepareHandoff(input: {
budget: BudgetTracker;
chain: string[];
currentSkill: string;
depth: number;
handoff: HandoffRequest;
history: ConversationMessage[];
request: AgentRequest;
response: AgentResponse;
}):
| {
budget: BudgetTracker;
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}
| null {
const target = this.resolveHandoffTarget(input);
const nextSkill = selectSkillForRequest(this.skills, {
...input.request,
skill: target.targetAgent,
trigger: "handoff",
});
if (!nextSkill) {
return null;
}
input.budget.recordHandoff(nextSkill.name);
metrics.handoffTotal.inc({
from: input.currentSkill,
reason: target.reason,
to: nextSkill.name,
});
const nextChain = [...input.chain, nextSkill.name];
const packet = buildHandoffPacket({
handoff: {
...input.handoff,
reason: target.reason,
targetAgent: nextSkill.name,
},
history: input.history,
request: input.request,
response: input.response,
sourceChain: nextChain,
state: {
source: input.currentSkill,
},
});
return {
budget: input.budget,
chain: nextChain,
depth: input.depth + 1,
history: input.history,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
handoffPacket: packet,
},
skill: nextSkill.name,
trigger: "handoff",
},
skill: nextSkill,
};
}
private resolveHandoffTarget(input: {
budget: BudgetTracker;
currentSkill: string;
handoff: HandoffRequest;
}): { reason: HandoffReason; targetAgent: string } {
const requestedTarget = input.handoff.targetAgent;
const allowed = this.transitionMatrix?.isAllowed(
input.currentSkill,
requestedTarget,
) ?? true;
if (!allowed) {
return {
reason: "policy-risk",
targetAgent: "supervisor",
};
}
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
if (verdict === "ok") {
return {
reason: input.handoff.reason,
targetAgent: requestedTarget,
};
}
if (verdict === "same-agent-loop") {
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
}
return {
reason: budgetVerdictToReason(verdict),
targetAgent: "supervisor",
};
}
private makeSupervisorFallback(
request: AgentRequest,
reason: HandoffReason,
): AgentResponse {
return {
confidence: 0.4,
metadata: {
supervisorFallbackReason: reason,
},
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
};
}
}
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
if (verdict === "same-agent-loop") {
return "same-agent-revisit";
}
if (verdict === "max-hops") {
return "max-hop";
}
return "policy-risk";
}
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
const value = response.metadata?.["handoffRequest"];
if (!value || typeof value !== "object") {
return null;
}
const handoff = value as Partial<HandoffRequest>;
if (
typeof handoff.targetAgent !== "string" ||
typeof handoff.reason !== "string" ||
typeof handoff.taskSpec !== "string" ||
typeof handoff.confidence !== "number" ||
typeof handoff.expectedOutput !== "string"
) {
return null;
}
return {
confidence: handoff.confidence,
evidenceRefs: Array.isArray(handoff.evidenceRefs)
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
: [],
expectedOutput: handoff.expectedOutput,
reason: handoff.reason,
risk:
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
? handoff.risk
: "medium",
targetAgent: handoff.targetAgent,
taskSpec: handoff.taskSpec,
...(handoff.userVisibleMessage
? { userVisibleMessage: handoff.userVisibleMessage }
: {}),
};
}
+77
View File
@@ -0,0 +1,77 @@
import { describe, expect, it } from "vitest";
import { BudgetTracker } from "./budget.js";
describe("BudgetTracker", () => {
it("allows handoff while within limits", () => {
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 1,
maxHandoffs: 2,
maxSameAgentVisits: 2,
maxToolCalls: 2,
});
expect(budget.assertBeforeHandoff("moderation")).toBe("ok");
budget.recordHandoff("moderation");
expect(budget.snapshot()).toMatchObject({
handoffs: 1,
visits: { moderation: 1 },
});
});
it("rejects loops and exhausted handoff budgets", () => {
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 1,
maxHandoffs: 1,
maxSameAgentVisits: 1,
maxToolCalls: 5,
});
budget.recordVisit("moderation");
expect(budget.assertBeforeHandoff("moderation")).toBe("same-agent-loop");
budget.recordHandoff("onboarding");
expect(budget.assertBeforeHandoff("supervisor")).toBe("max-hops");
});
it("rejects exhausted tool and cost budgets", () => {
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 0.5,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 1,
});
budget.recordToolCall();
expect(budget.assertBeforeHandoff("supervisor")).toBe("tool-budget");
const costBudget = new BudgetTracker({
enabled: true,
maxCostUsd: 0.5,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 5,
});
costBudget.recordToolCall(0.5);
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
});
it("can be disabled", () => {
const budget = new BudgetTracker({
enabled: false,
maxCostUsd: 0,
maxHandoffs: 0,
maxSameAgentVisits: 0,
maxToolCalls: 0,
});
budget.recordVisit("supervisor");
budget.recordHandoff("moderation");
budget.recordToolCall(1);
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
});
});
+88
View File
@@ -0,0 +1,88 @@
import {
getAgentBudgetEnabled,
getAgentBudgetMaxCostUsd,
getAgentBudgetMaxHandoffs,
getAgentBudgetMaxSameAgentVisits,
getAgentBudgetMaxToolCalls,
} from "../config.js";
export type BudgetVerdict =
| "ok"
| "max-hops"
| "same-agent-loop"
| "tool-budget"
| "cost-budget";
export interface BudgetSnapshot {
estimatedCostUsd: number;
handoffs: number;
maxCostUsd: number;
maxHandoffs: number;
maxSameAgentVisits: number;
maxToolCalls: number;
toolCalls: number;
visits: Record<string, number>;
}
export class BudgetTracker {
private estimatedCostUsd = 0;
private handoffs = 0;
private toolCalls = 0;
private readonly visits = new Map<string, number>();
constructor(
private readonly limits = {
enabled: getAgentBudgetEnabled(),
maxCostUsd: getAgentBudgetMaxCostUsd(),
maxHandoffs: getAgentBudgetMaxHandoffs(),
maxSameAgentVisits: getAgentBudgetMaxSameAgentVisits(),
maxToolCalls: getAgentBudgetMaxToolCalls(),
},
) {}
assertBeforeHandoff(target: string): BudgetVerdict {
if (!this.limits.enabled) {
return "ok";
}
if (this.handoffs >= this.limits.maxHandoffs) {
return "max-hops";
}
if ((this.visits.get(target) ?? 0) >= this.limits.maxSameAgentVisits) {
return "same-agent-loop";
}
if (this.toolCalls >= this.limits.maxToolCalls) {
return "tool-budget";
}
if (this.estimatedCostUsd >= this.limits.maxCostUsd) {
return "cost-budget";
}
return "ok";
}
recordHandoff(target: string): void {
this.handoffs += 1;
this.recordVisit(target);
}
recordToolCall(estimatedCostUsd = 0): void {
this.toolCalls += 1;
this.estimatedCostUsd += estimatedCostUsd;
}
recordVisit(target: string): void {
this.visits.set(target, (this.visits.get(target) ?? 0) + 1);
}
snapshot(): BudgetSnapshot {
return {
estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)),
handoffs: this.handoffs,
maxCostUsd: this.limits.maxCostUsd,
maxHandoffs: this.limits.maxHandoffs,
maxSameAgentVisits: this.limits.maxSameAgentVisits,
maxToolCalls: this.limits.maxToolCalls,
toolCalls: this.toolCalls,
visits: Object.fromEntries(this.visits.entries()),
};
}
}
+79
View File
@@ -0,0 +1,79 @@
import { describe, expect, it } from "vitest";
import { buildHandoffPacket, readHandoffPacket } from "./handoff-packet.js";
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
const request: AgentRequest = {
chatroomId: "room-1",
nickname: "tester",
requestId: "request-1",
skill: "onboarding",
trigger: "user-message",
userId: "user-1",
userMessage: "private original text",
};
const response: AgentResponse = {
confidence: 0.7,
requestId: "request-1",
source: "template",
text: "needs handoff",
};
const history: ConversationMessage[] = [
{
content: "이전 메시지",
role: "user",
timestamp: "2026-05-18T00:00:00.000Z",
},
];
describe("handoff packet", () => {
it("builds a bounded packet without raw user message", () => {
const packet = buildHandoffPacket({
handoff: {
confidence: 0.7,
evidenceRefs: ["tool-1"],
expectedOutput: "text",
reason: "out-of-scope",
risk: "medium",
targetAgent: "moderation",
taskSpec: "Call 010-1234-5678 after 2026-05-18",
},
history,
request,
response,
sourceChain: ["onboarding", "moderation"],
});
expect(packet.userGoal).toBe("Call <phone> after <date>");
expect(packet.userGoal).not.toContain(request.userMessage);
expect(packet.sourceChain).toEqual(["onboarding", "moderation"]);
expect(packet.evidence).toEqual([
{
id: "tool-1",
kind: "tool-call",
summary: "source agent evidence reference",
},
]);
});
it("reads packet from request context", () => {
const packet = buildHandoffPacket({
handoff: {
confidence: 0.7,
evidenceRefs: [],
expectedOutput: "text",
reason: "missing-tool",
risk: "low",
targetAgent: "moderation",
taskSpec: "continue",
},
request,
response,
sourceChain: ["onboarding", "moderation"],
});
expect(readHandoffPacket({ ...request, context: { handoffPacket: packet } })).toBe(packet);
});
});
+83
View File
@@ -0,0 +1,83 @@
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
HandoffReason,
HandoffRequest,
} from "./types.js";
export interface HandoffEvidence {
id: string;
kind: "message" | "tool-call" | "response";
summary: string;
}
export interface HandoffPacket {
constraints: string[];
currentState: Record<string, unknown>;
evidence: HandoffEvidence[];
expectedOutputSchema: string;
handoffReason: HandoffReason;
sourceChain: string[];
summary: string;
userGoal: string;
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function fallbackSummary(history: readonly ConversationMessage[]): string {
return history
.slice(-3)
.map((message) => `${message.role}: ${message.content.slice(0, 120)}`)
.join(" | ");
}
function sanitizeGoal(value: string | undefined): string {
if (!value?.trim()) {
return "이전 agent가 요청한 작업을 이어서 처리";
}
return value
.replace(/\b\d{2,4}[-./]\d{1,2}[-./]\d{1,2}\b/gu, "<date>")
.replace(/\b\d{2,3}[- ]?\d{3,4}[- ]?\d{4}\b/gu, "<phone>")
.slice(0, 240);
}
export function buildHandoffPacket(input: {
handoff: HandoffRequest;
history?: readonly ConversationMessage[] | undefined;
request: AgentRequest;
response: AgentResponse;
sourceChain: string[];
state?: Record<string, unknown> | undefined;
summary?: string | undefined;
}): HandoffPacket {
return {
constraints: [
"원문 사용자 메시지와 닉네임 등 PII를 새로 노출하지 않는다.",
"handoffRequest.expectedOutput 범위 안에서만 답한다.",
],
currentState: input.state ?? {},
evidence: input.handoff.evidenceRefs.map((id) => ({
id,
kind: "tool-call",
summary: "source agent evidence reference",
})),
expectedOutputSchema: input.handoff.expectedOutput,
handoffReason: input.handoff.reason,
sourceChain: input.sourceChain,
summary: input.summary?.trim() || fallbackSummary(input.history ?? []),
userGoal: sanitizeGoal(input.handoff.taskSpec),
};
}
export function readHandoffPacket(request: AgentRequest): HandoffPacket | undefined {
const packet = asRecord(request.context)["handoffPacket"];
if (!packet || typeof packet !== "object") {
return undefined;
}
return packet as HandoffPacket;
}
+21
View File
@@ -9,6 +9,27 @@ export interface AgentRequest {
context?: Record<string, unknown> | undefined;
}
export type HandoffReason =
| "out-of-scope"
| "missing-tool"
| "low-confidence"
| "validation-failed"
| "same-agent-revisit"
| "max-hop"
| "high-risk-action"
| "policy-risk";
export interface HandoffRequest {
targetAgent: string;
reason: HandoffReason;
taskSpec: string;
evidenceRefs: string[];
confidence: number;
risk: "low" | "medium" | "high";
expectedOutput: string;
userVisibleMessage?: string | undefined;
}
export interface AgentResponse {
requestId: string;
text: string;
+2 -1
View File
@@ -13,7 +13,8 @@ export interface AlertPayload {
| "eval-drift"
| "eval-score-drop"
| "latency"
| "llm-error-rate";
| "llm-error-rate"
| "supervisor-handoff";
metric: string;
severity: "info" | "warn" | "critical";
suggestedAction?: string | undefined;
+28
View File
@@ -98,6 +98,34 @@ export function getAgentRequestTimeoutMs(): number {
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
}
export function getAgentBudgetEnabled(): boolean {
return getBoolean("AGENT_BUDGET_ENABLED", true);
}
export function getAgentBudgetMaxHandoffs(): number {
return getPositiveNumber("AGENT_BUDGET_MAX_HANDOFFS", 4);
}
export function getAgentBudgetMaxSameAgentVisits(): number {
return getPositiveNumber("AGENT_BUDGET_MAX_SAME_AGENT_VISITS", 2);
}
export function getAgentBudgetMaxToolCalls(): number {
return getPositiveNumber("AGENT_BUDGET_MAX_TOOL_CALLS", 20);
}
export function getAgentBudgetMaxCostUsd(): number {
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
}
export function getAgentRuntimeVerifierEnabled(): boolean {
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
}
export function getMetricsPort(): number {
return getPositiveNumber("METRICS_PORT", 9464);
}
export function getDiscordVerifyTimeoutMs(): number {
return getPositiveNumber("DISCORD_VERIFY_TIMEOUT_MS", 2_500);
}
+40
View File
@@ -0,0 +1,40 @@
import type { AgentRequest, AgentResponse } from "@agent/types";
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
export type RuntimeVerdict = "pass" | "retry" | "escalate";
export interface RuntimeVerifierInput {
derived?: unknown;
packet?: unknown;
request: AgentRequest;
response: AgentResponse;
skill: string;
}
export function verifyOnboardingRuntime(
input: RuntimeVerifierInput,
): Promise<RuntimeVerdict> {
const derived = input.derived as DerivedState | undefined;
if (!derived) {
return Promise.resolve("pass");
}
const validation = validateOnboardingResponse(
input.response.text.trim() ? input.response.text : "[NO_RESPONSE]",
derived,
{
allowShortResponse: derived.allowShortResponse,
},
);
return Promise.resolve(validation.status === "reject" ? "retry" : "pass");
}
export function verifyModerationRuntime(
input: RuntimeVerifierInput,
): Promise<RuntimeVerdict> {
const text = input.response.text;
if (/https?:\/\//iu.test(text) || /\b\d{2,3}[- ]?\d{3,4}[- ]?\d{4}\b/u.test(text)) {
return Promise.resolve("escalate");
}
return Promise.resolve("pass");
}
+9
View File
@@ -6,6 +6,12 @@ import {
type ValidationResult,
} from "@skills/onboarding/runtime/validator";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
import {
verifyModerationRuntime,
verifyOnboardingRuntime,
type RuntimeVerifierInput,
type RuntimeVerdict,
} from "./runtime-verifier.js";
export type SkillValidationResult = ValidationResult;
@@ -14,6 +20,7 @@ export interface SkillJudgeConfig {
promptTemplate?: string | undefined;
toolRegistryFactory: () => ToolRegistry;
tournamentInstruction: string;
runtimeVerifier?: ((input: RuntimeVerifierInput) => Promise<RuntimeVerdict>) | undefined;
validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined;
weights?: Record<string, number> | undefined;
}
@@ -24,12 +31,14 @@ export const SKILL_JUDGE_CONFIGS: Record<string, SkillJudgeConfig> = {
toolRegistryFactory: createModerationToolRegistry,
tournamentInstruction:
"커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.",
runtimeVerifier: verifyModerationRuntime,
},
onboarding: {
axes: ["friendliness", "accuracy", "concision", "scope"],
toolRegistryFactory: createOnboardingToolRegistry,
tournamentInstruction:
"온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.",
runtimeVerifier: verifyOnboardingRuntime,
validator: (text, derived) =>
validateOnboardingResponse(text, derived as DerivedState, {
allowShortResponse: (derived as DerivedState).allowShortResponse,
+17
View File
@@ -0,0 +1,17 @@
const MODEL_RATES: Record<string, { inputPer1M: number; outputPer1M: number }> = {
"gpt-5.3-codex-spark": { inputPer1M: 0.15, outputPer1M: 0.6 },
"google/gemma-4-26b-a4b-it:free": { inputPer1M: 0, outputPer1M: 0 },
"google/gemma-4-31b-it:free": { inputPer1M: 0, outputPer1M: 0 },
};
export function estimateChatCost(
model: string,
tokensIn = 0,
tokensOut = 0,
): number {
const rate = MODEL_RATES[model] ?? { inputPer1M: 0.1, outputPer1M: 0.3 };
return Number(
((tokensIn / 1_000_000) * rate.inputPer1M +
(tokensOut / 1_000_000) * rate.outputPer1M).toFixed(6),
);
}
+9
View File
@@ -17,6 +17,8 @@ import {
import { LlmCircuitBreaker } from "./circuit.js";
import { getProviderAdapter } from "./providers/index.js";
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
import { estimateChatCost } from "./cost.js";
import { metrics } from "@obs/metrics";
const DEFAULT_MODEL = "Qwen3.5-4B";
@@ -211,6 +213,13 @@ export class LlmClient {
const payload = (await response.json()) as ChatCompletionResponse;
const result = provider.toResult(payload, endpoint.completionUrl);
metrics.tokenCostUsd.observe(
estimateChatCost(
request.model,
payload.usage?.prompt_tokens,
payload.usage?.completion_tokens,
),
);
span.setAttributes({
"gen_ai.response.finish_reasons": payload.choices
+11 -2
View File
@@ -23,12 +23,17 @@ const { Agent } = await import("./agent/index.js");
const { ConversationStore } = await import("./conversation/index.js");
const { createOnboardingSkill } = await import("./skills/onboarding/index.js");
const { createModerationSkill } = await import("./skills/moderation/index.js");
const { createSupervisorSkill } = await import("./skills/supervisor/index.js");
const { SkillRegistry } = await import("./skills/skill-loader.js");
const { TransitionMatrix } = await import("./skills/transition-matrix.js");
const { ShadowEvalBus } = await import("./eval/shadow-bus.js");
const { LlmClient } = await import("./llm/index.js");
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
const { RoomContextService } = await import("./room-context/index.js");
const { startMetricsServer, stopMetricsServer } = await import(
"./observability/metrics-server.js"
);
const { resolve } = await import("node:path");
function firstNonEmpty(
@@ -219,14 +224,16 @@ async function bootstrap(): Promise<void> {
// Agent setup
const conversationStore = new ConversationStore(garnetClient);
const agent = new Agent(natsClient, conversationStore);
const skillRegistry = SkillRegistry.load(resolve(process.cwd(), "src", "skills"));
const transitionMatrix = TransitionMatrix.fromRegistry(skillRegistry);
const agent = new Agent(natsClient, conversationStore, transitionMatrix);
// Register skills
const shadowEvalBus = new ShadowEvalBus(garnetClient);
const skillRegistry = SkillRegistry.load(resolve(process.cwd(), "src", "skills"));
const factories: Record<string, SkillFactory> = {
moderation: createModerationSkill,
onboarding: createOnboardingSkill,
supervisor: createSupervisorSkill,
};
const seenSkills = new Set<string>();
for (const manifest of skillRegistry.list()) {
@@ -260,12 +267,14 @@ async function bootstrap(): Promise<void> {
}
installShutdownHooks(async () => {
await stopMetricsServer();
await natsClient.close();
await garnetClient.close();
await closeMongoDB();
await shutdownTracing();
});
startMetricsServer();
await agent.start();
logger.info(`✓ ${serviceName} bootstrapped`);
+5
View File
@@ -3,10 +3,14 @@ import { AsyncLocalStorage } from 'node:async_hooks';
import type { Logger } from 'pino';
import { getLogger } from './logger.js';
import type { BudgetSnapshot } from '@agent/budget';
export interface RequestContext {
chatroomId: string;
deadlineAt?: string;
budget?: BudgetSnapshot | undefined;
handoffChain?: string[] | undefined;
handoffDepth?: number | undefined;
logger: Logger;
requestId: string;
skill: string;
@@ -27,6 +31,7 @@ export function runWithContext<T>(
chatroomId: context.chatroomId,
requestId: context.requestId,
skill: context.skill,
handoffDepth: context.handoffDepth,
trigger: context.trigger,
userId: context.userId,
});
+1
View File
@@ -5,6 +5,7 @@ import { getLogLevel, getNodeEnv, getServiceName } from '../config.js';
type RequestBindings = {
chatroomId?: string;
handoffDepth?: number | undefined;
requestId?: string;
skill?: string;
trigger?: string;
+49
View File
@@ -0,0 +1,49 @@
import { createServer, type Server } from "node:http";
import { getMetricsPort } from "../config.js";
import { getLogger } from "./logger.js";
import { renderMetrics } from "./metrics.js";
let server: Server | null = null;
export function startMetricsServer(port = getMetricsPort()): Server {
if (server) {
return server;
}
server = createServer((request, response) => {
if (request.url !== "/metrics") {
response.writeHead(404);
response.end("not found");
return;
}
response.writeHead(200, {
"Content-Type": "text/plain; version=0.0.4",
});
response.end(renderMetrics());
});
server.listen(port, () => {
getLogger().info({ port }, "✓ Metrics server listening");
});
return server;
}
export async function stopMetricsServer(): Promise<void> {
if (!server) {
return;
}
const current = server;
server = null;
await new Promise<void>((resolve, reject) => {
current.close((error) => {
if (error) {
reject(error);
return;
}
resolve();
});
});
}
+40
View File
@@ -0,0 +1,40 @@
import { describe, expect, it } from "vitest";
import { metrics, renderMetrics } from "./metrics.js";
describe("metrics", () => {
it("renders the agent metric families", () => {
const body = renderMetrics();
for (const name of [
"agent_routing_accuracy_total",
"agent_handoff_total",
"agent_handoff_loop_total",
"agent_handoff_count",
"agent_unnecessary_escalation_total",
"agent_missed_escalation_total",
"agent_token_cost_usd",
"agent_time_to_resolution_ms",
"agent_validator_fail_total",
"agent_human_override_total",
]) {
expect(body).toContain(`# HELP ${name}`);
}
});
it("renders counter labels and histogram observations", () => {
metrics.handoffTotal.inc({
from: "onboarding",
reason: "out-of-scope",
to: "moderation",
});
metrics.handoffCount.observe(2);
const body = renderMetrics();
expect(body).toContain(
'agent_handoff_total{from="onboarding",reason="out-of-scope",to="moderation"} 1',
);
expect(body).toContain('agent_handoff_count_bucket{le="2"} 1');
});
});
+134
View File
@@ -0,0 +1,134 @@
type Labels = Record<string, string | number | undefined>;
interface MetricSample {
labels: Labels;
value: number;
}
function labelKey(labels: Labels): string {
return JSON.stringify(
Object.fromEntries(
Object.entries(labels)
.filter(([, value]) => value !== undefined)
.sort(([left], [right]) => left.localeCompare(right)),
),
);
}
function formatLabels(labels: Labels): string {
const entries = Object.entries(labels).filter(([, value]) => value !== undefined);
if (entries.length === 0) {
return "";
}
return `{${entries
.map(([key, value]) => `${key}="${String(value).replace(/"/gu, '\\"')}"`)
.join(",")}}`;
}
class CounterMetric {
private readonly samples = new Map<string, MetricSample>();
constructor(
readonly name: string,
readonly help: string,
) {}
inc(labels: Labels = {}, value = 1): void {
const key = labelKey(labels);
const sample = this.samples.get(key) ?? { labels, value: 0 };
sample.value += value;
this.samples.set(key, sample);
}
render(): string[] {
return [
`# HELP ${this.name} ${this.help}`,
`# TYPE ${this.name} counter`,
...[...this.samples.values()].map(
(sample) => `${this.name}${formatLabels(sample.labels)} ${sample.value}`,
),
];
}
}
class HistogramMetric {
private readonly values: number[] = [];
constructor(
readonly name: string,
readonly help: string,
private readonly buckets: number[],
) {}
observe(value: number): void {
if (Number.isFinite(value)) {
this.values.push(value);
}
}
render(): string[] {
const lines = [
`# HELP ${this.name} ${this.help}`,
`# TYPE ${this.name} histogram`,
];
for (const bucket of this.buckets) {
const count = this.values.filter((value) => value <= bucket).length;
lines.push(`${this.name}_bucket{le="${bucket}"} ${count}`);
}
lines.push(`${this.name}_bucket{le="+Inf"} ${this.values.length}`);
lines.push(
`${this.name}_sum ${this.values.reduce((sum, value) => sum + value, 0)}`,
);
lines.push(`${this.name}_count ${this.values.length}`);
return lines;
}
}
export const metrics = {
handoffTotal: new CounterMetric("agent_handoff_total", "Agent handoffs"),
handoffLoopTotal: new CounterMetric(
"agent_handoff_loop_total",
"Agent handoff loops",
),
humanOverrideTotal: new CounterMetric(
"agent_human_override_total",
"Human override events",
),
missedEscalationTotal: new CounterMetric(
"agent_missed_escalation_total",
"Missed escalations",
),
routingAccuracyTotal: new CounterMetric(
"agent_routing_accuracy_total",
"Eval routing accuracy",
),
unnecessaryEscalationTotal: new CounterMetric(
"agent_unnecessary_escalation_total",
"Unnecessary escalations",
),
validatorFailTotal: new CounterMetric(
"agent_validator_fail_total",
"Validator failures",
),
handoffCount: new HistogramMetric(
"agent_handoff_count",
"Handoffs per request",
[0, 1, 2, 3, 4],
),
timeToResolutionMs: new HistogramMetric(
"agent_time_to_resolution_ms",
"Time to resolution",
[100, 500, 1_000, 2_500, 5_000, 10_000],
),
tokenCostUsd: new HistogramMetric(
"agent_token_cost_usd",
"Estimated LLM token cost USD",
[0.001, 0.01, 0.05, 0.1, 0.5, 1],
),
};
export function renderMetrics(): string {
return Object.values(metrics)
.flatMap((metric) => metric.render())
.join("\n");
}
+62
View File
@@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import { parseControlTokens } from "./control-token-parser.js";
describe("parseControlTokens", () => {
it("extracts handoff control tokens and strips them from text", () => {
const parsed = parseControlTokens(
"잠시 확인할게요. [HANDOFF:moderation:policy-risk]",
);
expect(parsed.stripped).toBe("잠시 확인할게요.");
expect(parsed.tokens).toEqual([
{
kind: "handoff",
raw: "[HANDOFF:moderation:policy-risk]",
reason: "policy-risk",
targetAgent: "moderation",
},
]);
});
it("extracts no-response tokens", () => {
const parsed = parseControlTokens("[NO_RESPONSE]");
expect(parsed.stripped).toBe("");
expect(parsed.tokens).toEqual([
{
kind: "no-response",
raw: "[NO_RESPONSE]",
},
]);
});
it("keeps invalid handoff reasons visible", () => {
const parsed = parseControlTokens("[HANDOFF:moderation:not-a-reason]");
expect(parsed.stripped).toBe("[HANDOFF:moderation:not-a-reason]");
expect(parsed.tokens).toEqual([]);
});
it("accepts all handoff reason codes", () => {
const reasons = [
"out-of-scope",
"missing-tool",
"low-confidence",
"validation-failed",
"same-agent-revisit",
"max-hop",
"high-risk-action",
"policy-risk",
];
for (const reason of reasons) {
const parsed = parseControlTokens(`[HANDOFF:supervisor:${reason}]`);
expect(parsed.tokens[0]).toMatchObject({
kind: "handoff",
reason,
targetAgent: "supervisor",
});
}
});
});
+55
View File
@@ -0,0 +1,55 @@
import type { HandoffReason } from "@agent/types";
const HANDOFF_REASONS = new Set<HandoffReason>([
"out-of-scope",
"missing-tool",
"low-confidence",
"validation-failed",
"same-agent-revisit",
"max-hop",
"high-risk-action",
"policy-risk",
]);
export type ControlToken =
| { kind: "no-response"; raw: string }
| {
kind: "handoff";
raw: string;
reason: HandoffReason;
targetAgent: string;
};
export function parseControlTokens(text: string): {
stripped: string;
tokens: ControlToken[];
} {
const tokens: ControlToken[] = [];
let stripped = text;
stripped = stripped.replace(/\[NO_RESPONSE\]/gu, (raw) => {
tokens.push({ kind: "no-response", raw });
return "";
});
stripped = stripped.replace(
/\[HANDOFF:([a-z0-9_-]+):([a-z-]+)\]/giu,
(raw, target: string, reason: string) => {
if (!HANDOFF_REASONS.has(reason as HandoffReason)) {
return raw;
}
tokens.push({
kind: "handoff",
raw,
reason: reason as HandoffReason,
targetAgent: target,
});
return "";
},
);
return {
stripped: stripped.trim(),
tokens,
};
}
+95
View File
@@ -0,0 +1,95 @@
import { describe, expect, it } from "vitest";
import type { HandoffReason } from "@agent/types";
import { decide } from "./trigger.js";
describe("escalation trigger", () => {
it("turns explicit handoff reasons into structured requests", () => {
const reasons: HandoffReason[] = [
"out-of-scope",
"missing-tool",
"low-confidence",
"validation-failed",
"same-agent-revisit",
"max-hop",
"high-risk-action",
"policy-risk",
];
for (const reason of reasons) {
const decision = decide({
classified: { confidence: 0.8 },
reason,
targetAgent: "supervisor",
taskSpec: "review",
});
expect(decision).toMatchObject({
reason,
shouldHandoff: true,
handoff: {
reason,
targetAgent: "supervisor",
taskSpec: "review",
},
});
}
});
it("escalates exhausted budgets and validation rejects", () => {
expect(
decide({
budget: {
estimatedCostUsd: 0,
handoffs: 4,
maxCostUsd: 0.5,
maxHandoffs: 4,
maxSameAgentVisits: 2,
maxToolCalls: 20,
toolCalls: 0,
visits: {},
},
}).handoff?.reason,
).toBe("max-hop");
expect(decide({ validation: { status: "reject" } }).handoff?.reason).toBe(
"validation-failed",
);
});
it("escalates low-confidence and uncertain high-risk responses", () => {
expect(
decide({ classified: { confidence: 0.2 } }).handoff?.reason,
).toBe("low-confidence");
expect(
decide({
capabilityCard: {
canHandle: ["policy"],
cannotHandle: [],
domain: "moderation",
handoffTargets: ["supervisor"],
outputContract: { format: "text", maxChars: "500" },
riskTier: "high",
},
classified: { confidence: 0.6 },
}).handoff?.reason,
).toBe("high-risk-action");
});
it("does not escalate ordinary confident responses", () => {
expect(
decide({
capabilityCard: {
canHandle: ["help"],
cannotHandle: [],
domain: "onboarding",
handoffTargets: ["supervisor"],
outputContract: { format: "text", maxChars: "500" },
riskTier: "medium",
},
classified: { confidence: 0.9 },
}),
).toEqual({ shouldHandoff: false });
});
});
+56
View File
@@ -0,0 +1,56 @@
import type { HandoffReason, HandoffRequest } from "@agent/types";
import type { BudgetSnapshot } from "@agent/budget";
import type { SkillCapabilityCard } from "@skills/skill-loader";
export interface EscalationDecision {
handoff?: HandoffRequest | undefined;
reason?: HandoffReason | undefined;
shouldHandoff: boolean;
}
export function decide(input: {
budget?: BudgetSnapshot | undefined;
capabilityCard?: SkillCapabilityCard | undefined;
classified?: { confidence?: number | undefined } | undefined;
reason?: HandoffReason | undefined;
targetAgent?: string | undefined;
taskSpec?: string | undefined;
validation?: { status?: string | undefined } | undefined;
}): EscalationDecision {
const confidence = input.classified?.confidence;
let reason: HandoffReason | undefined = input.reason;
if (!reason) {
if (input.budget && input.budget.handoffs >= input.budget.maxHandoffs) {
reason = "max-hop";
} else if (input.validation?.status === "reject") {
reason = "validation-failed";
} else if (typeof confidence === "number" && confidence < 0.35) {
reason = "low-confidence";
} else if (
input.capabilityCard?.riskTier === "high" &&
typeof confidence === "number" &&
confidence < 0.7
) {
reason = "high-risk-action";
}
}
if (!reason) {
return { shouldHandoff: false };
}
return {
handoff: {
confidence: confidence ?? 0.5,
evidenceRefs: [],
expectedOutput: "text",
reason,
risk: input.capabilityCard?.riskTier ?? "medium",
targetAgent: input.targetAgent ?? "supervisor",
taskSpec: input.taskSpec ?? "Escalate this request to the appropriate skill.",
},
reason,
shouldHandoff: true,
};
}
+9
View File
@@ -3,6 +3,15 @@ name: moderation
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
version: 1
priority: 50
domain: moderation
canHandle: [rules, violation-report, appeal, sanctions, policy-risk]
cannotHandle: [onboarding-status, nickname-format, photo-verification, discord-onboarding]
handoffTargets: [onboarding, supervisor]
riskTier: high
outputContract:
format: text
maxChars: "500"
requiredFields: text
intents: [greeting, rule-question, violation-report, appeal, off-topic]
triggers: [user-message]
tools: [lookup-rule]
+23
View File
@@ -6,6 +6,7 @@ import type {
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader";
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
@@ -22,6 +23,24 @@ type RuleTopic =
| "sanctions"
| "spam";
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: ["rules", "violation-report", "appeal", "sanctions", "policy-risk"],
cannotHandle: [
"onboarding-status",
"nickname-format",
"photo-verification",
"discord-onboarding",
],
domain: "moderation",
handoffTargets: ["onboarding", "supervisor"],
outputContract: {
format: "text",
maxChars: "500",
requiredFields: "text",
},
riskTier: "high",
};
function formatHistory(history: ConversationMessage[]): string {
if (history.length === 0) {
return "(none)";
@@ -156,6 +175,10 @@ export class ModerationSkill implements Skill {
return request.skill === this.name;
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
conversationHistory: ConversationMessage[],
+9
View File
@@ -3,6 +3,15 @@ name: onboarding
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
version: 1
priority: 70
domain: onboarding
canHandle: [new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed, onboarding-help]
cannotHandle: [moderation-report, sanctions, appeal, policy-enforcement]
handoffTargets: [moderation, supervisor]
riskTier: medium
outputContract:
format: text
maxChars: "500"
requiredFields: text
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
@@ -79,4 +79,14 @@ describe("validateOnboardingResponse", () => {
expect(result.status).toBe("pass");
});
it("passes handoff control tokens", () => {
const result = validateOnboardingResponse(
"[HANDOFF:moderation:policy-risk]",
makeDerived(),
);
expect(result.status).toBe("pass");
expect(result.reasons).toContain("handoff");
});
});
+16 -6
View File
@@ -1,5 +1,6 @@
import { trace } from "@opentelemetry/api";
import { getMaxResponseLength } from "../../../config.js";
import { parseControlTokens } from "../../control-token-parser.js";
import { VALID_TIER_SET } from "../constants.js";
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
import type { DerivedState } from "../state/onboarding-state.js";
@@ -131,15 +132,12 @@ export function validateOnboardingResponse(
derived: DerivedState,
options?: { allowShortResponse?: boolean | undefined },
): ValidationResult {
const text = postProcessResponse(draft);
const parsedTokens = parseControlTokens(postProcessResponse(draft));
const text = parsedTokens.stripped;
const rejectReasons: string[] = [];
const softWarnReasons: string[] = [];
if (!text) {
rejectReasons.push("empty");
}
if (text.includes("[NO_RESPONSE]")) {
if (parsedTokens.tokens.some((token) => token.kind === "no-response")) {
return markValidation({
status: "pass",
text: "[NO_RESPONSE]",
@@ -147,6 +145,18 @@ export function validateOnboardingResponse(
});
}
if (parsedTokens.tokens.some((token) => token.kind === "handoff")) {
return markValidation({
status: "pass",
text,
reasons: ["handoff"],
});
}
if (!text) {
rejectReasons.push("empty");
}
if (!options?.allowShortResponse && text.length < 10) {
rejectReasons.push("minLength");
}
+31
View File
@@ -5,6 +5,7 @@ import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import type { SkillCapabilityCard } from '@skills/skill-loader';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { createOnboardingToolRegistry } from './tools.js';
@@ -68,6 +69,32 @@ import type {
StoredOnboardingSession,
} from './types.js';
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: [
'new-member',
'nickname-invalid',
'photo-failed',
'progress-update',
'timeout-warning',
'completed',
'onboarding-help',
],
cannotHandle: [
'moderation-report',
'sanctions',
'appeal',
'policy-enforcement',
],
domain: 'onboarding',
handoffTargets: ['moderation', 'supervisor'],
outputContract: {
format: 'text',
maxChars: '500',
requiredFields: 'text',
},
riskTier: 'medium',
};
export class OnboardingSkill implements Skill {
readonly name = 'onboarding';
readonly priority = 70;
@@ -99,6 +126,10 @@ export class OnboardingSkill implements Skill {
return request.skill === 'onboarding';
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
conversationHistory: ConversationMessage[],
+19
View File
@@ -21,6 +21,15 @@ function makeTempSkillRoot(): string {
"priority: 65",
"intents: [hello, help]",
"triggers: [user-message]",
"domain: demo-domain",
"canHandle: [demo-task]",
"cannotHandle: [other-task]",
"handoffTargets: [supervisor]",
"riskTier: medium",
"outputContract:",
" format: text",
" maxChars: \"500\"",
" requiredFields: text",
"tools: [demo-tool]",
"resources:",
" guide: ./resources/guide.md",
@@ -41,8 +50,18 @@ describe("SkillRegistry", () => {
expect(manifest).toMatchObject({
name: "demo",
description: "Demo skill.",
canHandle: ["demo-task"],
cannotHandle: ["other-task"],
domain: "demo-domain",
handoffTargets: ["supervisor"],
intents: ["hello", "help"],
outputContract: {
format: "text",
maxChars: "500",
requiredFields: "text",
},
priority: 65,
riskTier: "medium",
tools: ["demo-tool"],
trustTier: "T2",
version: 1,
+55 -1
View File
@@ -3,14 +3,36 @@ import { dirname, resolve } from "node:path";
import { getLogger } from "@obs/logger";
export type TrustTier = "T1" | "T2" | "T3" | "T4";
export type SkillRiskTier = "low" | "medium" | "high";
export interface SkillOutputContract {
format?: string | undefined;
maxChars?: string | undefined;
requiredFields?: string | undefined;
}
export interface SkillCapabilityCard {
canHandle: string[];
cannotHandle: string[];
domain: string;
handoffTargets: string[];
outputContract: SkillOutputContract;
riskTier: SkillRiskTier;
}
export interface SkillManifest {
bodyPath: string;
canHandle: string[];
cannotHandle: string[];
description: string;
domain: string;
handoffTargets: string[];
intents: string[];
name: string;
outputContract: SkillOutputContract;
priority: number;
resources: Record<string, string>;
riskTier: SkillRiskTier;
tools: string[];
triggers: string[];
trustTier: TrustTier;
@@ -121,10 +143,32 @@ function readResources(
);
}
function readStringMap(
fields: Record<string, unknown>,
key: string,
): Record<string, string> {
const raw = fields[key];
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
return {};
}
return Object.fromEntries(
Object.entries(raw as Record<string, unknown>).flatMap(([mapKey, value]) =>
typeof value === "string" && value.trim()
? [[mapKey, value.trim().replace(/^["']|["']$/gu, "")]]
: [],
),
);
}
function isTrustTier(value: unknown): value is TrustTier {
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
}
function isRiskTier(value: unknown): value is SkillRiskTier {
return value === "low" || value === "medium" || value === "high";
}
function isPriority(value: unknown): value is number {
return (
typeof value === "number" &&
@@ -147,6 +191,8 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
const priority = parsed.fields["priority"] ?? 50;
const version = parsed.fields["version"];
const trustTier = parsed.fields["trustTier"];
const domain = parsed.fields["domain"];
const riskTier = parsed.fields["riskTier"] ?? "low";
if (
typeof name !== "string" ||
@@ -155,20 +201,28 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
!description.trim() ||
!isPriority(priority) ||
typeof version !== "number" ||
!isTrustTier(trustTier)
!isTrustTier(trustTier) ||
!isRiskTier(riskTier)
) {
return null;
}
const outputContract = readStringMap(parsed.fields, "outputContract");
return {
body: parsed.body,
manifest: {
bodyPath: path,
canHandle: readStringArray(parsed.fields, "canHandle"),
cannotHandle: readStringArray(parsed.fields, "cannotHandle"),
description,
domain: typeof domain === "string" && domain.trim() ? domain : "general",
handoffTargets: readStringArray(parsed.fields, "handoffTargets"),
intents: readStringArray(parsed.fields, "intents"),
name,
outputContract,
priority,
resources: readResources(parsed.fields, skillDir),
riskTier,
tools: readStringArray(parsed.fields, "tools"),
triggers: readStringArray(parsed.fields, "triggers"),
trustTier,
+2
View File
@@ -3,10 +3,12 @@ import type {
AgentResponse,
ConversationMessage,
} from "@agent/types";
import type { SkillCapabilityCard } from "./skill-loader.js";
export interface Skill {
readonly name: string;
readonly priority: number;
capabilityCard?(): SkillCapabilityCard;
canHandle(request: AgentRequest): boolean;
handle(
request: AgentRequest,
+22
View File
@@ -0,0 +1,22 @@
---
name: supervisor
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
version: 1
priority: 10
domain: meta
canHandle: [*]
cannotHandle: []
handoffTargets: [onboarding, moderation]
riskTier: medium
outputContract:
format: text
maxChars: "500"
requiredFields: text
intents: []
triggers: [handoff, user-message]
tools: []
resources:
trustTier: T4
---
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
+9
View File
@@ -0,0 +1,9 @@
import type { Skill } from "@skills/skill";
import type { SkillDeps } from "@skills/skill-factory";
import { SupervisorSkill } from "./skill.js";
export { SupervisorSkill } from "./skill.js";
export function createSupervisorSkill(deps: SkillDeps): Skill {
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
}
@@ -0,0 +1,7 @@
export function getIntentKinds(): string[] {
return [];
}
export function getToolNames(): string[] {
return [];
}
+14
View File
@@ -0,0 +1,14 @@
import type { GarnetClient } from "@common/garnet";
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
export async function appendSupervisorReview(
garnetClient: Pick<GarnetClient, "appendStream">,
fields: Record<string, string>,
): Promise<void> {
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
...fields,
type: "supervisor-handoff",
queuedAt: new Date().toISOString(),
});
}
+17
View File
@@ -0,0 +1,17 @@
import type { HandoffPacket } from "@agent/handoff-packet";
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
const text = [
packet?.userGoal,
packet?.summary,
packet?.handoffReason,
].join(" ");
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) {
return "moderation";
}
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) {
return "onboarding";
}
return null;
}
+90
View File
@@ -0,0 +1,90 @@
import { describe, expect, it } from "vitest";
import type { AlertPayload } from "@common/nats-subjects";
import { buildHandoffPacket } from "@agent/handoff-packet";
import type { AgentRequest, AgentResponse } from "@agent/types";
import { SupervisorSkill } from "./skill.js";
class FakeGarnet {
readonly streams: Array<{ key: string; fields: Record<string, unknown> }> = [];
appendStream(key: string, fields: Record<string, unknown>): Promise<string> {
this.streams.push({ key, fields });
return Promise.resolve("1-0");
}
setStringIfAbsent(): Promise<boolean> {
return Promise.resolve(true);
}
}
class FakeNats {
readonly published: AlertPayload[] = [];
publish<T = unknown>(_subject: string, payload: T): Promise<void> {
this.published.push(payload as AlertPayload);
return Promise.resolve();
}
}
const baseRequest: AgentRequest = {
chatroomId: "room-1",
nickname: "tester",
requestId: "request-1",
skill: "supervisor",
trigger: "handoff",
userId: "user-1",
};
const sourceResponse: AgentResponse = {
confidence: 0.3,
requestId: "request-1",
source: "template",
text: "needs review",
};
describe("SupervisorSkill", () => {
it("queues review and publishes alert for handoffs", async () => {
const garnet = new FakeGarnet();
const nats = new FakeNats();
const skill = new SupervisorSkill(garnet, nats);
const packet = buildHandoffPacket({
handoff: {
confidence: 0.3,
evidenceRefs: [],
expectedOutput: "text",
reason: "max-hop",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "가입 상태 확인",
},
request: baseRequest,
response: sourceResponse,
sourceChain: ["onboarding", "supervisor"],
});
const response = await skill.handle(
{ ...baseRequest, context: { handoffPacket: packet } },
[],
);
expect(response.text).toContain("운영자");
expect(response.metadata).toMatchObject({
supervisorReason: "max-hop",
suggestedTarget: "onboarding",
});
expect(garnet.streams[0]).toMatchObject({
key: "lolympus-agent:review-queue",
fields: {
reason: "max-hop",
requestId: "request-1",
target: "onboarding",
type: "supervisor-handoff",
},
});
expect(nats.published[0]).toMatchObject({
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
});
});
});
+93
View File
@@ -0,0 +1,93 @@
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
} from "@agent/types";
import { readHandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats";
import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader";
import { appendSupervisorReview } from "./review-queue.js";
import { routeSupervisorTarget } from "./routing-rules.js";
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: ["*"],
cannotHandle: [],
domain: "meta",
handoffTargets: ["onboarding", "moderation"],
outputContract: {
format: "text",
maxChars: "500",
requiredFields: "text",
},
riskTier: "medium",
};
export class SupervisorSkill implements Skill {
readonly name = "supervisor";
readonly priority = 10;
private readonly alertService: AlertService;
constructor(
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
natsClient: Pick<NatsClient, "publish">,
) {
this.alertService = new AlertService(garnetClient, natsClient);
}
canHandle(request: AgentRequest): boolean {
return request.skill === this.name;
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
_conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const packet = readHandoffPacket(request);
const target = routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope";
await appendSupervisorReview(this.garnetClient, {
chatroomId: request.chatroomId,
reason,
requestId: request.requestId,
target: target ?? "human-review",
userId: request.userId,
});
metrics.humanOverrideTotal.inc({ kind: reason });
await this.alertService.sendAlert(
{
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
severity: "warn",
suggestedAction: "Review the supervisor handoff queue.",
threshold: 0,
timestamp: new Date().toISOString(),
value: 1,
window: "request",
},
{
dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`,
},
);
return {
confidence: 0.6,
metadata: {
supervisorReason: reason,
suggestedTarget: target,
},
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
};
}
}
+65
View File
@@ -0,0 +1,65 @@
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { SkillRegistry } from "./skill-loader.js";
import { TransitionMatrix } from "./transition-matrix.js";
function writeSkill(
root: string,
name: string,
handoffTargets: string[],
): void {
const skillDir = join(root, name);
mkdirSync(skillDir, { recursive: true });
writeFileSync(
join(skillDir, "SKILL.md"),
[
"---",
`name: ${name}`,
`description: ${name} skill.`,
"version: 1",
"priority: 50",
"domain: test",
"canHandle: [test]",
"cannotHandle: []",
`handoffTargets: [${handoffTargets.join(", ")}]`,
"riskTier: low",
"outputContract:",
" format: text",
" maxChars: \"500\"",
"trustTier: T2",
"---",
"",
"body",
].join("\n"),
);
}
describe("TransitionMatrix", () => {
it("allows declared handoffs and blocks self loops", () => {
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-"));
writeSkill(root, "onboarding", ["moderation"]);
writeSkill(root, "moderation", []);
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
expect(matrix.isAllowed("onboarding", "moderation")).toBe(true);
expect(matrix.isAllowed("onboarding", "supervisor")).toBe(true);
expect(matrix.isAllowed("onboarding", "onboarding")).toBe(false);
expect(matrix.isAllowed("moderation", "onboarding")).toBe(false);
});
it("reports unknown handoff targets", () => {
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-bad-"));
writeSkill(root, "onboarding", ["missing"]);
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
expect(matrix.validateGraph(["onboarding"])).toContain(
"onboarding: unknown handoff target missing",
);
});
});
+47
View File
@@ -0,0 +1,47 @@
import type { SkillRegistry } from "./skill-loader.js";
export class TransitionMatrix {
private constructor(private readonly transitions: Map<string, Set<string>>) {}
static fromRegistry(registry: SkillRegistry): TransitionMatrix {
const transitions = new Map<string, Set<string>>();
for (const manifest of registry.list()) {
const targets =
manifest.name === "supervisor"
? manifest.handoffTargets
: [...manifest.handoffTargets, "supervisor"];
transitions.set(
manifest.name,
new Set(targets),
);
}
return new TransitionMatrix(transitions);
}
isAllowed(from: string, to: string): boolean {
if (from === to) {
return false;
}
return this.transitions.get(from)?.has(to) ?? true;
}
targetsOf(from: string): string[] {
return [...(this.transitions.get(from) ?? new Set<string>())];
}
validateGraph(knownSkills?: readonly string[]): string[] {
const known = new Set([...(knownSkills ?? this.transitions.keys()), "supervisor"]);
const errors: string[] = [];
for (const [from, targets] of this.transitions.entries()) {
if (targets.has(from)) {
errors.push(`${from}: handoffTargets must not include itself`);
}
for (const target of targets) {
if (!known.has(target)) {
errors.push(`${from}: unknown handoff target ${target}`);
}
}
}
return errors;
}
}
+48 -1
View File
@@ -3,8 +3,10 @@ import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
import { TransitionMatrix } from "./transition-matrix.js";
import * as moderationBinding from "./moderation/manifest-binding.js";
import * as onboardingBinding from "./onboarding/manifest-binding.js";
import * as supervisorBinding from "./supervisor/manifest-binding.js";
interface ManifestBinding {
getIntentKinds(): string[];
@@ -14,6 +16,7 @@ interface ManifestBinding {
const BINDINGS: Record<string, ManifestBinding> = {
moderation: moderationBinding,
onboarding: onboardingBinding,
supervisor: supervisorBinding,
};
function sortUnique(values: readonly string[]): string[] {
@@ -32,12 +35,15 @@ function diff(left: readonly string[], right: readonly string[]): string[] {
function validateManifest(
manifest: SkillManifest,
binding: ManifestBinding | undefined,
knownSkills: readonly string[],
): string[] {
if (!binding) {
return [`${manifest.name}: missing manifest binding`];
}
const errors: string[] = [];
const capabilityErrors = validateCapabilityCard(manifest, knownSkills);
errors.push(...capabilityErrors);
const manifestIntents = sortUnique(manifest.intents);
const codeIntents = sortUnique(binding.getIntentKinds());
const unknownIntents = diff(manifestIntents, codeIntents);
@@ -66,6 +72,45 @@ function validateManifest(
return errors;
}
function validateCapabilityCard(
manifest: SkillManifest,
knownSkills: readonly string[],
): string[] {
const errors: string[] = [];
const canHandle = new Set(manifest.canHandle);
const overlap = manifest.cannotHandle.filter((item) => canHandle.has(item));
if (overlap.length > 0) {
errors.push(
`${manifest.name}: canHandle and cannotHandle overlap: ${format(overlap)}`,
);
}
const riskTier: string = manifest.riskTier;
if (riskTier !== "low" && riskTier !== "medium" && riskTier !== "high") {
errors.push(`${manifest.name}: invalid riskTier ${riskTier}`);
}
const knownTargets = new Set([...knownSkills, "supervisor"]);
const missingTargets = manifest.handoffTargets.filter(
(target) => !knownTargets.has(target),
);
if (missingTargets.length > 0) {
errors.push(
`${manifest.name}: unknown handoffTargets: ${format(missingTargets)}`,
);
}
if (manifest.handoffTargets.includes(manifest.name)) {
errors.push(`${manifest.name}: handoffTargets must not include itself`);
}
if (!manifest.domain.trim()) {
errors.push(`${manifest.name}: domain is required`);
}
if (!manifest.outputContract.format || !manifest.outputContract.maxChars) {
errors.push(
`${manifest.name}: outputContract.format and outputContract.maxChars are required`,
);
}
return errors;
}
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
const byPriority = new Map<number, string[]>();
for (const manifest of manifests) {
@@ -84,11 +129,13 @@ function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
const registry = SkillRegistry.load(rootDir);
const manifests = registry.list();
const knownSkills = manifests.map((manifest) => manifest.name);
return [
...manifests.flatMap((manifest) =>
validateManifest(manifest, BINDINGS[manifest.name]),
validateManifest(manifest, BINDINGS[manifest.name], knownSkills),
),
...validatePriorityConflicts(manifests),
...TransitionMatrix.fromRegistry(registry).validateGraph(knownSkills),
];
}