Add bounded skill handoff graph
This commit is contained in:
1 parent
54bc295ead
commit
c14adac162
42 files changed
+2243
-31
No files matched your search
+249
-1
@@ -1,7 +1,10 @@
|
||||
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import { selectSkillForRequest } from "./agent.js";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||
import { Agent, selectSkillForRequest } from "./agent.js";
|
||||
|
||||
function makeRequest(skill = "shared"): AgentRequest {
|
||||
return {
|
||||
@@ -32,6 +35,60 @@ function makeSkill(name: string, priority: number, canHandle = true): Skill {
|
||||
};
|
||||
}
|
||||
|
||||
class MemoryConversationStore {
|
||||
readonly messages: ConversationMessage[] = [];
|
||||
|
||||
getHistory(): Promise<ConversationMessage[]> {
|
||||
return Promise.resolve([...this.messages]);
|
||||
}
|
||||
|
||||
appendMessage(
|
||||
_chatroomId: string,
|
||||
_userId: string,
|
||||
message: ConversationMessage,
|
||||
): Promise<void> {
|
||||
this.messages.push(message);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class FakeNatsClient {
|
||||
readonly responses: AgentResponse[] = [];
|
||||
|
||||
respond(_msg: unknown, response: AgentResponse): Promise<void> {
|
||||
this.responses.push(response);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleThroughAgent(
|
||||
skills: readonly Skill[],
|
||||
request: AgentRequest,
|
||||
transitionMatrix?: TransitionMatrix,
|
||||
): Promise<{
|
||||
nats: FakeNatsClient;
|
||||
store: MemoryConversationStore;
|
||||
}> {
|
||||
const nats = new FakeNatsClient();
|
||||
const store = new MemoryConversationStore();
|
||||
const agent = new Agent(
|
||||
nats as unknown as NatsClient,
|
||||
store as unknown as ConversationStore,
|
||||
transitionMatrix,
|
||||
);
|
||||
for (const skill of skills) {
|
||||
agent.registerSkill(skill);
|
||||
}
|
||||
|
||||
await (agent as unknown as {
|
||||
handleRequest(msg: { data: Uint8Array }): Promise<void>;
|
||||
}).handleRequest({
|
||||
data: new TextEncoder().encode(JSON.stringify(request)),
|
||||
});
|
||||
|
||||
return { nats, store };
|
||||
}
|
||||
|
||||
describe("selectSkillForRequest", () => {
|
||||
it("selects the highest-priority matching skill", () => {
|
||||
const request = makeRequest();
|
||||
@@ -51,3 +108,194 @@ describe("selectSkillForRequest", () => {
|
||||
expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Agent handoff dispatch", () => {
|
||||
it("responds once with the final handoff target response", async () => {
|
||||
const source: Skill = {
|
||||
...makeSkill("source", 50),
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.8,
|
||||
metadata: {
|
||||
handoffRequest: {
|
||||
confidence: 0.8,
|
||||
evidenceRefs: ["rule-1"],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "target",
|
||||
taskSpec: "continue",
|
||||
},
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "handoff",
|
||||
}),
|
||||
};
|
||||
const target: Skill = {
|
||||
...makeSkill("target", 60),
|
||||
canHandle: (request) => request.skill === "target",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 1,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "final answer",
|
||||
}),
|
||||
};
|
||||
|
||||
const { nats, store } = await handleThroughAgent(
|
||||
[source, target],
|
||||
makeRequest("source"),
|
||||
);
|
||||
|
||||
expect(nats.responses).toHaveLength(1);
|
||||
expect(nats.responses[0]?.text).toBe("final answer");
|
||||
expect(store.messages.map((message) => message.content)).toEqual([
|
||||
"final answer",
|
||||
]);
|
||||
});
|
||||
|
||||
it("routes handoff control tokens through metadata", async () => {
|
||||
const source: Skill = {
|
||||
...makeSkill("source", 50),
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.9,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "[HANDOFF:target:missing-tool]",
|
||||
}),
|
||||
};
|
||||
const target: Skill = {
|
||||
...makeSkill("target", 60),
|
||||
canHandle: (request) => request.skill === "target",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 1,
|
||||
metadata: {
|
||||
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "handled elsewhere",
|
||||
}),
|
||||
};
|
||||
|
||||
const { nats } = await handleThroughAgent(
|
||||
[source, target],
|
||||
makeRequest("source"),
|
||||
);
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("handled elsewhere");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
});
|
||||
|
||||
it("routes runtime verifier escalations to supervisor", async () => {
|
||||
const previous = process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
|
||||
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = "true";
|
||||
try {
|
||||
const moderation: Skill = {
|
||||
...makeSkill("moderation", 50),
|
||||
canHandle: (request) => request.skill === "moderation",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.9,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "https://not-allowed.example",
|
||||
}),
|
||||
};
|
||||
const supervisor: Skill = {
|
||||
...makeSkill("supervisor", 90),
|
||||
canHandle: (request) => request.skill === "supervisor",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "sent to review",
|
||||
}),
|
||||
};
|
||||
|
||||
const { nats } = await handleThroughAgent(
|
||||
[moderation, supervisor],
|
||||
makeRequest("moderation"),
|
||||
);
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("sent to review");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
} finally {
|
||||
if (previous === undefined) {
|
||||
delete process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
|
||||
} else {
|
||||
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = previous;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("reroutes illegal transitions to supervisor", async () => {
|
||||
const source: Skill = {
|
||||
...makeSkill("source", 50),
|
||||
canHandle: (request) => request.skill === "source",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.9,
|
||||
metadata: {
|
||||
handoffRequest: {
|
||||
confidence: 0.9,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "target",
|
||||
taskSpec: "continue",
|
||||
},
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "handoff",
|
||||
}),
|
||||
};
|
||||
const target: Skill = {
|
||||
...makeSkill("target", 60),
|
||||
canHandle: (request) => request.skill === "target",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 1,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "should not run",
|
||||
}),
|
||||
};
|
||||
const supervisor: Skill = {
|
||||
...makeSkill("supervisor", 10),
|
||||
canHandle: (request) => request.skill === "supervisor",
|
||||
handle: (request) =>
|
||||
Promise.resolve({
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "supervisor review",
|
||||
}),
|
||||
};
|
||||
|
||||
const matrix = {
|
||||
isAllowed: () => false,
|
||||
} as unknown as TransitionMatrix;
|
||||
const { nats } = await handleThroughAgent(
|
||||
[source, target, supervisor],
|
||||
makeRequest("source"),
|
||||
matrix,
|
||||
);
|
||||
|
||||
expect(nats.responses[0]?.text).toBe("supervisor review");
|
||||
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||
});
|
||||
});
|
||||
+447
-19
@@ -5,8 +5,22 @@ import { getRequestLogger, runWithContext } from "@obs/context";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import { withSkillSpan, withSpan } from "@obs/spans";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { AgentRequest, AgentResponse } from "./types.js";
|
||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||
import { parseControlTokens } from "@skills/control-token-parser";
|
||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
|
||||
import { buildHandoffPacket } from "./handoff-packet.js";
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
HandoffRequest,
|
||||
} from "./types.js";
|
||||
import {
|
||||
getAgentRuntimeVerifierEnabled,
|
||||
getAgentRequestTimeoutMs,
|
||||
getRequestSubject,
|
||||
getSafetyMarginMs,
|
||||
@@ -43,6 +57,7 @@ export class Agent {
|
||||
constructor(
|
||||
private readonly natsClient: NatsClient,
|
||||
private readonly conversationStore: ConversationStore,
|
||||
private readonly transitionMatrix?: TransitionMatrix | undefined,
|
||||
) {}
|
||||
|
||||
registerSkill(skill: Skill): void {
|
||||
@@ -107,18 +122,28 @@ export class Agent {
|
||||
"[Agent] Request received",
|
||||
);
|
||||
|
||||
const skill = selectSkillForRequest(this.skills, request);
|
||||
let skill = selectSkillForRequest(this.skills, request);
|
||||
if (!skill) {
|
||||
logger.info("[Agent] No skill found for request");
|
||||
const response: AgentResponse = {
|
||||
requestId: request.requestId,
|
||||
text: "",
|
||||
confidence: 0,
|
||||
source: "template",
|
||||
const supervisorRequest: AgentRequest = {
|
||||
...request,
|
||||
skill: "supervisor",
|
||||
trigger: "handoff",
|
||||
};
|
||||
rootSpan.setAttribute("agent.source", "template");
|
||||
await this.natsClient.respond(msg, response);
|
||||
return;
|
||||
skill = selectSkillForRequest(this.skills, supervisorRequest);
|
||||
if (skill) {
|
||||
request = supervisorRequest;
|
||||
} else {
|
||||
logger.info("[Agent] No skill found for request");
|
||||
const response: AgentResponse = {
|
||||
requestId: request.requestId,
|
||||
text: "",
|
||||
confidence: 0,
|
||||
source: "template",
|
||||
};
|
||||
rootSpan.setAttribute("agent.source", "template");
|
||||
await this.natsClient.respond(msg, response);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -139,14 +164,17 @@ export class Agent {
|
||||
);
|
||||
}
|
||||
|
||||
const response = await withSkillSpan(
|
||||
skill.name,
|
||||
{
|
||||
"agent.skill": skill.name,
|
||||
"agent.trigger": request.trigger,
|
||||
},
|
||||
() => skill.handle(request, history),
|
||||
);
|
||||
const startedAt = Date.now();
|
||||
const budget = new BudgetTracker();
|
||||
budget.recordVisit(skill.name);
|
||||
const response = await this.dispatch({
|
||||
budget,
|
||||
chain: [skill.name],
|
||||
depth: 0,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
});
|
||||
|
||||
if (response.text.trim()) {
|
||||
await this.conversationStore.appendMessage(
|
||||
@@ -161,6 +189,8 @@ export class Agent {
|
||||
}
|
||||
|
||||
await this.natsClient.respond(msg, response);
|
||||
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
|
||||
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
|
||||
rootSpan.setAttribute(
|
||||
"agent.source",
|
||||
response.text.trim() ? response.source : "empty",
|
||||
@@ -200,4 +230,402 @@ export class Agent {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async dispatch(input: {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
const { budget, chain, depth, history, request, skill } = input;
|
||||
const contextBase = {
|
||||
chatroomId: request.chatroomId,
|
||||
deadlineAt: new Date(
|
||||
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
||||
).toISOString(),
|
||||
requestId: request.requestId,
|
||||
skill: skill.name,
|
||||
startedAt: new Date().toISOString(),
|
||||
trigger: request.trigger,
|
||||
userId: request.userId,
|
||||
};
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
...contextBase,
|
||||
budget: budget.snapshot(),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
},
|
||||
async () => {
|
||||
const response = await this.runSkillOnce({
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
});
|
||||
const normalized = this.normalizeControlTokens(response, skill.name);
|
||||
const verified = await this.applyRuntimeVerifier({
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
response: normalized,
|
||||
skill,
|
||||
});
|
||||
const escalated = this.applyEscalationTrigger({
|
||||
budget,
|
||||
request,
|
||||
response: verified,
|
||||
skill,
|
||||
});
|
||||
|
||||
const handoff = readHandoffRequest(escalated);
|
||||
if (!handoff) {
|
||||
return escalated;
|
||||
}
|
||||
|
||||
const next = this.prepareHandoff({
|
||||
budget,
|
||||
chain,
|
||||
currentSkill: skill.name,
|
||||
depth,
|
||||
handoff,
|
||||
history,
|
||||
request,
|
||||
response: escalated,
|
||||
});
|
||||
|
||||
if (!next) {
|
||||
return this.makeSupervisorFallback(request, handoff.reason);
|
||||
}
|
||||
|
||||
return this.dispatch(next);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async runSkillOnce(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
return withSkillSpan(
|
||||
input.skill.name,
|
||||
{
|
||||
"agent.handoff_chain": input.chain.join(">"),
|
||||
"agent.handoff_depth": input.depth,
|
||||
"agent.skill": input.skill.name,
|
||||
"agent.trigger": input.request.trigger,
|
||||
},
|
||||
() => input.skill.handle(input.request, input.history),
|
||||
);
|
||||
}
|
||||
|
||||
private normalizeControlTokens(
|
||||
response: AgentResponse,
|
||||
currentSkill: string,
|
||||
): AgentResponse {
|
||||
const parsed = parseControlTokens(response.text);
|
||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||
const metadata = { ...(response.metadata ?? {}) };
|
||||
if (handoff?.kind === "handoff") {
|
||||
metadata.handoffRequest = {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: handoff.reason,
|
||||
risk: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: `Continue handling from ${currentSkill}`,
|
||||
} satisfies HandoffRequest;
|
||||
}
|
||||
return {
|
||||
...response,
|
||||
metadata,
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
private async applyRuntimeVerifier(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
if (!getAgentRuntimeVerifierEnabled()) {
|
||||
return input.response;
|
||||
}
|
||||
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
|
||||
if (!verifier) {
|
||||
return input.response;
|
||||
}
|
||||
const verdict = await verifier({
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (verdict === "pass") {
|
||||
return input.response;
|
||||
}
|
||||
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
|
||||
if (verdict === "retry" && input.depth === 0) {
|
||||
const retryResponse = await this.runSkillOnce({
|
||||
...input,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
runtimeVerifierRetry: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
const normalizedRetry = this.normalizeControlTokens(
|
||||
retryResponse,
|
||||
input.skill.name,
|
||||
);
|
||||
if (readHandoffRequest(normalizedRetry)) {
|
||||
return normalizedRetry;
|
||||
}
|
||||
const retryVerdict = await verifier({
|
||||
request: input.request,
|
||||
response: normalizedRetry,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (retryVerdict === "pass") {
|
||||
return {
|
||||
...normalizedRetry,
|
||||
metadata: {
|
||||
...(normalizedRetry.metadata ?? {}),
|
||||
runtimeVerifier: "retry-pass",
|
||||
},
|
||||
};
|
||||
}
|
||||
metrics.validatorFailTotal.inc({
|
||||
skill: input.skill.name,
|
||||
stage: `retry-${retryVerdict}`,
|
||||
});
|
||||
return this.withRuntimeVerifierHandoff(normalizedRetry);
|
||||
}
|
||||
return this.withRuntimeVerifierHandoff(input.response);
|
||||
}
|
||||
|
||||
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "validation-failed",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "Runtime verifier rejected the response.",
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private applyEscalationTrigger(input: {
|
||||
budget: BudgetTracker;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
}): AgentResponse {
|
||||
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
const decision = decideEscalation({
|
||||
budget: input.budget.snapshot(),
|
||||
capabilityCard: input.skill.capabilityCard?.(),
|
||||
classified: { confidence: input.response.confidence },
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
||||
});
|
||||
if (!decision.handoff) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
return {
|
||||
...input.response,
|
||||
metadata: {
|
||||
...(input.response.metadata ?? {}),
|
||||
handoffRequest: decision.handoff,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private prepareHandoff(input: {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
currentSkill: string;
|
||||
depth: number;
|
||||
handoff: HandoffRequest;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}):
|
||||
| {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}
|
||||
| null {
|
||||
const target = this.resolveHandoffTarget(input);
|
||||
const nextSkill = selectSkillForRequest(this.skills, {
|
||||
...input.request,
|
||||
skill: target.targetAgent,
|
||||
trigger: "handoff",
|
||||
});
|
||||
if (!nextSkill) {
|
||||
return null;
|
||||
}
|
||||
|
||||
input.budget.recordHandoff(nextSkill.name);
|
||||
metrics.handoffTotal.inc({
|
||||
from: input.currentSkill,
|
||||
reason: target.reason,
|
||||
to: nextSkill.name,
|
||||
});
|
||||
|
||||
const nextChain = [...input.chain, nextSkill.name];
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
...input.handoff,
|
||||
reason: target.reason,
|
||||
targetAgent: nextSkill.name,
|
||||
},
|
||||
history: input.history,
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
sourceChain: nextChain,
|
||||
state: {
|
||||
source: input.currentSkill,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
budget: input.budget,
|
||||
chain: nextChain,
|
||||
depth: input.depth + 1,
|
||||
history: input.history,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
handoffPacket: packet,
|
||||
},
|
||||
skill: nextSkill.name,
|
||||
trigger: "handoff",
|
||||
},
|
||||
skill: nextSkill,
|
||||
};
|
||||
}
|
||||
|
||||
private resolveHandoffTarget(input: {
|
||||
budget: BudgetTracker;
|
||||
currentSkill: string;
|
||||
handoff: HandoffRequest;
|
||||
}): { reason: HandoffReason; targetAgent: string } {
|
||||
const requestedTarget = input.handoff.targetAgent;
|
||||
const allowed = this.transitionMatrix?.isAllowed(
|
||||
input.currentSkill,
|
||||
requestedTarget,
|
||||
) ?? true;
|
||||
if (!allowed) {
|
||||
return {
|
||||
reason: "policy-risk",
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
||||
if (verdict === "ok") {
|
||||
return {
|
||||
reason: input.handoff.reason,
|
||||
targetAgent: requestedTarget,
|
||||
};
|
||||
}
|
||||
|
||||
if (verdict === "same-agent-loop") {
|
||||
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
|
||||
}
|
||||
|
||||
return {
|
||||
reason: budgetVerdictToReason(verdict),
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
private makeSupervisorFallback(
|
||||
request: AgentRequest,
|
||||
reason: HandoffReason,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: 0.4,
|
||||
metadata: {
|
||||
supervisorFallbackReason: reason,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||
if (verdict === "same-agent-loop") {
|
||||
return "same-agent-revisit";
|
||||
}
|
||||
if (verdict === "max-hops") {
|
||||
return "max-hop";
|
||||
}
|
||||
return "policy-risk";
|
||||
}
|
||||
|
||||
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
||||
const value = response.metadata?.["handoffRequest"];
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const handoff = value as Partial<HandoffRequest>;
|
||||
if (
|
||||
typeof handoff.targetAgent !== "string" ||
|
||||
typeof handoff.reason !== "string" ||
|
||||
typeof handoff.taskSpec !== "string" ||
|
||||
typeof handoff.confidence !== "number" ||
|
||||
typeof handoff.expectedOutput !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
confidence: handoff.confidence,
|
||||
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
||||
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||
: [],
|
||||
expectedOutput: handoff.expectedOutput,
|
||||
reason: handoff.reason,
|
||||
risk:
|
||||
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
||||
? handoff.risk
|
||||
: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: handoff.taskSpec,
|
||||
...(handoff.userVisibleMessage
|
||||
? { userVisibleMessage: handoff.userVisibleMessage }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { BudgetTracker } from "./budget.js";
|
||||
|
||||
describe("BudgetTracker", () => {
|
||||
it("allows handoff while within limits", () => {
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: 1,
|
||||
maxHandoffs: 2,
|
||||
maxSameAgentVisits: 2,
|
||||
maxToolCalls: 2,
|
||||
});
|
||||
|
||||
expect(budget.assertBeforeHandoff("moderation")).toBe("ok");
|
||||
budget.recordHandoff("moderation");
|
||||
expect(budget.snapshot()).toMatchObject({
|
||||
handoffs: 1,
|
||||
visits: { moderation: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects loops and exhausted handoff budgets", () => {
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: 1,
|
||||
maxHandoffs: 1,
|
||||
maxSameAgentVisits: 1,
|
||||
maxToolCalls: 5,
|
||||
});
|
||||
|
||||
budget.recordVisit("moderation");
|
||||
expect(budget.assertBeforeHandoff("moderation")).toBe("same-agent-loop");
|
||||
|
||||
budget.recordHandoff("onboarding");
|
||||
expect(budget.assertBeforeHandoff("supervisor")).toBe("max-hops");
|
||||
});
|
||||
|
||||
it("rejects exhausted tool and cost budgets", () => {
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: 0.5,
|
||||
maxHandoffs: 5,
|
||||
maxSameAgentVisits: 5,
|
||||
maxToolCalls: 1,
|
||||
});
|
||||
|
||||
budget.recordToolCall();
|
||||
expect(budget.assertBeforeHandoff("supervisor")).toBe("tool-budget");
|
||||
|
||||
const costBudget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: 0.5,
|
||||
maxHandoffs: 5,
|
||||
maxSameAgentVisits: 5,
|
||||
maxToolCalls: 5,
|
||||
});
|
||||
costBudget.recordToolCall(0.5);
|
||||
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
|
||||
});
|
||||
|
||||
it("can be disabled", () => {
|
||||
const budget = new BudgetTracker({
|
||||
enabled: false,
|
||||
maxCostUsd: 0,
|
||||
maxHandoffs: 0,
|
||||
maxSameAgentVisits: 0,
|
||||
maxToolCalls: 0,
|
||||
});
|
||||
|
||||
budget.recordVisit("supervisor");
|
||||
budget.recordHandoff("moderation");
|
||||
budget.recordToolCall(1);
|
||||
|
||||
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
import {
|
||||
getAgentBudgetEnabled,
|
||||
getAgentBudgetMaxCostUsd,
|
||||
getAgentBudgetMaxHandoffs,
|
||||
getAgentBudgetMaxSameAgentVisits,
|
||||
getAgentBudgetMaxToolCalls,
|
||||
} from "../config.js";
|
||||
|
||||
export type BudgetVerdict =
|
||||
| "ok"
|
||||
| "max-hops"
|
||||
| "same-agent-loop"
|
||||
| "tool-budget"
|
||||
| "cost-budget";
|
||||
|
||||
export interface BudgetSnapshot {
|
||||
estimatedCostUsd: number;
|
||||
handoffs: number;
|
||||
maxCostUsd: number;
|
||||
maxHandoffs: number;
|
||||
maxSameAgentVisits: number;
|
||||
maxToolCalls: number;
|
||||
toolCalls: number;
|
||||
visits: Record<string, number>;
|
||||
}
|
||||
|
||||
export class BudgetTracker {
|
||||
private estimatedCostUsd = 0;
|
||||
private handoffs = 0;
|
||||
private toolCalls = 0;
|
||||
private readonly visits = new Map<string, number>();
|
||||
|
||||
constructor(
|
||||
private readonly limits = {
|
||||
enabled: getAgentBudgetEnabled(),
|
||||
maxCostUsd: getAgentBudgetMaxCostUsd(),
|
||||
maxHandoffs: getAgentBudgetMaxHandoffs(),
|
||||
maxSameAgentVisits: getAgentBudgetMaxSameAgentVisits(),
|
||||
maxToolCalls: getAgentBudgetMaxToolCalls(),
|
||||
},
|
||||
) {}
|
||||
|
||||
assertBeforeHandoff(target: string): BudgetVerdict {
|
||||
if (!this.limits.enabled) {
|
||||
return "ok";
|
||||
}
|
||||
if (this.handoffs >= this.limits.maxHandoffs) {
|
||||
return "max-hops";
|
||||
}
|
||||
if ((this.visits.get(target) ?? 0) >= this.limits.maxSameAgentVisits) {
|
||||
return "same-agent-loop";
|
||||
}
|
||||
if (this.toolCalls >= this.limits.maxToolCalls) {
|
||||
return "tool-budget";
|
||||
}
|
||||
if (this.estimatedCostUsd >= this.limits.maxCostUsd) {
|
||||
return "cost-budget";
|
||||
}
|
||||
return "ok";
|
||||
}
|
||||
|
||||
recordHandoff(target: string): void {
|
||||
this.handoffs += 1;
|
||||
this.recordVisit(target);
|
||||
}
|
||||
|
||||
recordToolCall(estimatedCostUsd = 0): void {
|
||||
this.toolCalls += 1;
|
||||
this.estimatedCostUsd += estimatedCostUsd;
|
||||
}
|
||||
|
||||
recordVisit(target: string): void {
|
||||
this.visits.set(target, (this.visits.get(target) ?? 0) + 1);
|
||||
}
|
||||
|
||||
snapshot(): BudgetSnapshot {
|
||||
return {
|
||||
estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)),
|
||||
handoffs: this.handoffs,
|
||||
maxCostUsd: this.limits.maxCostUsd,
|
||||
maxHandoffs: this.limits.maxHandoffs,
|
||||
maxSameAgentVisits: this.limits.maxSameAgentVisits,
|
||||
maxToolCalls: this.limits.maxToolCalls,
|
||||
toolCalls: this.toolCalls,
|
||||
visits: Object.fromEntries(this.visits.entries()),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { buildHandoffPacket, readHandoffPacket } from "./handoff-packet.js";
|
||||
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
|
||||
|
||||
const request: AgentRequest = {
|
||||
chatroomId: "room-1",
|
||||
nickname: "tester",
|
||||
requestId: "request-1",
|
||||
skill: "onboarding",
|
||||
trigger: "user-message",
|
||||
userId: "user-1",
|
||||
userMessage: "private original text",
|
||||
};
|
||||
|
||||
const response: AgentResponse = {
|
||||
confidence: 0.7,
|
||||
requestId: "request-1",
|
||||
source: "template",
|
||||
text: "needs handoff",
|
||||
};
|
||||
|
||||
const history: ConversationMessage[] = [
|
||||
{
|
||||
content: "이전 메시지",
|
||||
role: "user",
|
||||
timestamp: "2026-05-18T00:00:00.000Z",
|
||||
},
|
||||
];
|
||||
|
||||
describe("handoff packet", () => {
|
||||
it("builds a bounded packet without raw user message", () => {
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
confidence: 0.7,
|
||||
evidenceRefs: ["tool-1"],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "moderation",
|
||||
taskSpec: "Call 010-1234-5678 after 2026-05-18",
|
||||
},
|
||||
history,
|
||||
request,
|
||||
response,
|
||||
sourceChain: ["onboarding", "moderation"],
|
||||
});
|
||||
|
||||
expect(packet.userGoal).toBe("Call <phone> after <date>");
|
||||
expect(packet.userGoal).not.toContain(request.userMessage);
|
||||
expect(packet.sourceChain).toEqual(["onboarding", "moderation"]);
|
||||
expect(packet.evidence).toEqual([
|
||||
{
|
||||
id: "tool-1",
|
||||
kind: "tool-call",
|
||||
summary: "source agent evidence reference",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads packet from request context", () => {
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
confidence: 0.7,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "missing-tool",
|
||||
risk: "low",
|
||||
targetAgent: "moderation",
|
||||
taskSpec: "continue",
|
||||
},
|
||||
request,
|
||||
response,
|
||||
sourceChain: ["onboarding", "moderation"],
|
||||
});
|
||||
|
||||
expect(readHandoffPacket({ ...request, context: { handoffPacket: packet } })).toBe(packet);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
HandoffRequest,
|
||||
} from "./types.js";
|
||||
|
||||
export interface HandoffEvidence {
|
||||
id: string;
|
||||
kind: "message" | "tool-call" | "response";
|
||||
summary: string;
|
||||
}
|
||||
|
||||
export interface HandoffPacket {
|
||||
constraints: string[];
|
||||
currentState: Record<string, unknown>;
|
||||
evidence: HandoffEvidence[];
|
||||
expectedOutputSchema: string;
|
||||
handoffReason: HandoffReason;
|
||||
sourceChain: string[];
|
||||
summary: string;
|
||||
userGoal: string;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function fallbackSummary(history: readonly ConversationMessage[]): string {
|
||||
return history
|
||||
.slice(-3)
|
||||
.map((message) => `${message.role}: ${message.content.slice(0, 120)}`)
|
||||
.join(" | ");
|
||||
}
|
||||
|
||||
function sanitizeGoal(value: string | undefined): string {
|
||||
if (!value?.trim()) {
|
||||
return "이전 agent가 요청한 작업을 이어서 처리";
|
||||
}
|
||||
return value
|
||||
.replace(/\b\d{2,4}[-./]\d{1,2}[-./]\d{1,2}\b/gu, "<date>")
|
||||
.replace(/\b\d{2,3}[- ]?\d{3,4}[- ]?\d{4}\b/gu, "<phone>")
|
||||
.slice(0, 240);
|
||||
}
|
||||
|
||||
export function buildHandoffPacket(input: {
|
||||
handoff: HandoffRequest;
|
||||
history?: readonly ConversationMessage[] | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
sourceChain: string[];
|
||||
state?: Record<string, unknown> | undefined;
|
||||
summary?: string | undefined;
|
||||
}): HandoffPacket {
|
||||
return {
|
||||
constraints: [
|
||||
"원문 사용자 메시지와 닉네임 등 PII를 새로 노출하지 않는다.",
|
||||
"handoffRequest.expectedOutput 범위 안에서만 답한다.",
|
||||
],
|
||||
currentState: input.state ?? {},
|
||||
evidence: input.handoff.evidenceRefs.map((id) => ({
|
||||
id,
|
||||
kind: "tool-call",
|
||||
summary: "source agent evidence reference",
|
||||
})),
|
||||
expectedOutputSchema: input.handoff.expectedOutput,
|
||||
handoffReason: input.handoff.reason,
|
||||
sourceChain: input.sourceChain,
|
||||
summary: input.summary?.trim() || fallbackSummary(input.history ?? []),
|
||||
userGoal: sanitizeGoal(input.handoff.taskSpec),
|
||||
};
|
||||
}
|
||||
|
||||
export function readHandoffPacket(request: AgentRequest): HandoffPacket | undefined {
|
||||
const packet = asRecord(request.context)["handoffPacket"];
|
||||
if (!packet || typeof packet !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
return packet as HandoffPacket;
|
||||
}
|
||||
@@ -9,6 +9,27 @@ export interface AgentRequest {
|
||||
context?: Record<string, unknown> | undefined;
|
||||
}
|
||||
|
||||
export type HandoffReason =
|
||||
| "out-of-scope"
|
||||
| "missing-tool"
|
||||
| "low-confidence"
|
||||
| "validation-failed"
|
||||
| "same-agent-revisit"
|
||||
| "max-hop"
|
||||
| "high-risk-action"
|
||||
| "policy-risk";
|
||||
|
||||
export interface HandoffRequest {
|
||||
targetAgent: string;
|
||||
reason: HandoffReason;
|
||||
taskSpec: string;
|
||||
evidenceRefs: string[];
|
||||
confidence: number;
|
||||
risk: "low" | "medium" | "high";
|
||||
expectedOutput: string;
|
||||
userVisibleMessage?: string | undefined;
|
||||
}
|
||||
|
||||
export interface AgentResponse {
|
||||
requestId: string;
|
||||
text: string;
|
||||
|
||||
@@ -13,7 +13,8 @@ export interface AlertPayload {
|
||||
| "eval-drift"
|
||||
| "eval-score-drop"
|
||||
| "latency"
|
||||
| "llm-error-rate";
|
||||
| "llm-error-rate"
|
||||
| "supervisor-handoff";
|
||||
metric: string;
|
||||
severity: "info" | "warn" | "critical";
|
||||
suggestedAction?: string | undefined;
|
||||
|
||||
@@ -98,6 +98,34 @@ export function getAgentRequestTimeoutMs(): number {
|
||||
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
|
||||
}
|
||||
|
||||
export function getAgentBudgetEnabled(): boolean {
|
||||
return getBoolean("AGENT_BUDGET_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getAgentBudgetMaxHandoffs(): number {
|
||||
return getPositiveNumber("AGENT_BUDGET_MAX_HANDOFFS", 4);
|
||||
}
|
||||
|
||||
export function getAgentBudgetMaxSameAgentVisits(): number {
|
||||
return getPositiveNumber("AGENT_BUDGET_MAX_SAME_AGENT_VISITS", 2);
|
||||
}
|
||||
|
||||
export function getAgentBudgetMaxToolCalls(): number {
|
||||
return getPositiveNumber("AGENT_BUDGET_MAX_TOOL_CALLS", 20);
|
||||
}
|
||||
|
||||
export function getAgentBudgetMaxCostUsd(): number {
|
||||
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
|
||||
}
|
||||
|
||||
export function getAgentRuntimeVerifierEnabled(): boolean {
|
||||
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
|
||||
}
|
||||
|
||||
export function getMetricsPort(): number {
|
||||
return getPositiveNumber("METRICS_PORT", 9464);
|
||||
}
|
||||
|
||||
export function getDiscordVerifyTimeoutMs(): number {
|
||||
return getPositiveNumber("DISCORD_VERIFY_TIMEOUT_MS", 2_500);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
export type RuntimeVerdict = "pass" | "retry" | "escalate";
|
||||
|
||||
export interface RuntimeVerifierInput {
|
||||
derived?: unknown;
|
||||
packet?: unknown;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: string;
|
||||
}
|
||||
|
||||
export function verifyOnboardingRuntime(
|
||||
input: RuntimeVerifierInput,
|
||||
): Promise<RuntimeVerdict> {
|
||||
const derived = input.derived as DerivedState | undefined;
|
||||
if (!derived) {
|
||||
return Promise.resolve("pass");
|
||||
}
|
||||
const validation = validateOnboardingResponse(
|
||||
input.response.text.trim() ? input.response.text : "[NO_RESPONSE]",
|
||||
derived,
|
||||
{
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
},
|
||||
);
|
||||
return Promise.resolve(validation.status === "reject" ? "retry" : "pass");
|
||||
}
|
||||
|
||||
export function verifyModerationRuntime(
|
||||
input: RuntimeVerifierInput,
|
||||
): Promise<RuntimeVerdict> {
|
||||
const text = input.response.text;
|
||||
if (/https?:\/\//iu.test(text) || /\b\d{2,3}[- ]?\d{3,4}[- ]?\d{4}\b/u.test(text)) {
|
||||
return Promise.resolve("escalate");
|
||||
}
|
||||
return Promise.resolve("pass");
|
||||
}
|
||||
@@ -6,6 +6,12 @@ import {
|
||||
type ValidationResult,
|
||||
} from "@skills/onboarding/runtime/validator";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
verifyModerationRuntime,
|
||||
verifyOnboardingRuntime,
|
||||
type RuntimeVerifierInput,
|
||||
type RuntimeVerdict,
|
||||
} from "./runtime-verifier.js";
|
||||
|
||||
export type SkillValidationResult = ValidationResult;
|
||||
|
||||
@@ -14,6 +20,7 @@ export interface SkillJudgeConfig {
|
||||
promptTemplate?: string | undefined;
|
||||
toolRegistryFactory: () => ToolRegistry;
|
||||
tournamentInstruction: string;
|
||||
runtimeVerifier?: ((input: RuntimeVerifierInput) => Promise<RuntimeVerdict>) | undefined;
|
||||
validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined;
|
||||
weights?: Record<string, number> | undefined;
|
||||
}
|
||||
@@ -24,12 +31,14 @@ export const SKILL_JUDGE_CONFIGS: Record<string, SkillJudgeConfig> = {
|
||||
toolRegistryFactory: createModerationToolRegistry,
|
||||
tournamentInstruction:
|
||||
"커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.",
|
||||
runtimeVerifier: verifyModerationRuntime,
|
||||
},
|
||||
onboarding: {
|
||||
axes: ["friendliness", "accuracy", "concision", "scope"],
|
||||
toolRegistryFactory: createOnboardingToolRegistry,
|
||||
tournamentInstruction:
|
||||
"온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.",
|
||||
runtimeVerifier: verifyOnboardingRuntime,
|
||||
validator: (text, derived) =>
|
||||
validateOnboardingResponse(text, derived as DerivedState, {
|
||||
allowShortResponse: (derived as DerivedState).allowShortResponse,
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
const MODEL_RATES: Record<string, { inputPer1M: number; outputPer1M: number }> = {
|
||||
"gpt-5.3-codex-spark": { inputPer1M: 0.15, outputPer1M: 0.6 },
|
||||
"google/gemma-4-26b-a4b-it:free": { inputPer1M: 0, outputPer1M: 0 },
|
||||
"google/gemma-4-31b-it:free": { inputPer1M: 0, outputPer1M: 0 },
|
||||
};
|
||||
|
||||
export function estimateChatCost(
|
||||
model: string,
|
||||
tokensIn = 0,
|
||||
tokensOut = 0,
|
||||
): number {
|
||||
const rate = MODEL_RATES[model] ?? { inputPer1M: 0.1, outputPer1M: 0.3 };
|
||||
return Number(
|
||||
((tokensIn / 1_000_000) * rate.inputPer1M +
|
||||
(tokensOut / 1_000_000) * rate.outputPer1M).toFixed(6),
|
||||
);
|
||||
}
|
||||
@@ -17,6 +17,8 @@ import {
|
||||
import { LlmCircuitBreaker } from "./circuit.js";
|
||||
import { getProviderAdapter } from "./providers/index.js";
|
||||
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
|
||||
import { estimateChatCost } from "./cost.js";
|
||||
import { metrics } from "@obs/metrics";
|
||||
|
||||
const DEFAULT_MODEL = "Qwen3.5-4B";
|
||||
|
||||
@@ -211,6 +213,13 @@ export class LlmClient {
|
||||
|
||||
const payload = (await response.json()) as ChatCompletionResponse;
|
||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||
metrics.tokenCostUsd.observe(
|
||||
estimateChatCost(
|
||||
request.model,
|
||||
payload.usage?.prompt_tokens,
|
||||
payload.usage?.completion_tokens,
|
||||
),
|
||||
);
|
||||
|
||||
span.setAttributes({
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
|
||||
+11
-2
@@ -23,12 +23,17 @@ const { Agent } = await import("./agent/index.js");
|
||||
const { ConversationStore } = await import("./conversation/index.js");
|
||||
const { createOnboardingSkill } = await import("./skills/onboarding/index.js");
|
||||
const { createModerationSkill } = await import("./skills/moderation/index.js");
|
||||
const { createSupervisorSkill } = await import("./skills/supervisor/index.js");
|
||||
const { SkillRegistry } = await import("./skills/skill-loader.js");
|
||||
const { TransitionMatrix } = await import("./skills/transition-matrix.js");
|
||||
const { ShadowEvalBus } = await import("./eval/shadow-bus.js");
|
||||
const { LlmClient } = await import("./llm/index.js");
|
||||
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
|
||||
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
|
||||
const { RoomContextService } = await import("./room-context/index.js");
|
||||
const { startMetricsServer, stopMetricsServer } = await import(
|
||||
"./observability/metrics-server.js"
|
||||
);
|
||||
const { resolve } = await import("node:path");
|
||||
|
||||
function firstNonEmpty(
|
||||
@@ -219,14 +224,16 @@ async function bootstrap(): Promise<void> {
|
||||
|
||||
// Agent setup
|
||||
const conversationStore = new ConversationStore(garnetClient);
|
||||
const agent = new Agent(natsClient, conversationStore);
|
||||
const skillRegistry = SkillRegistry.load(resolve(process.cwd(), "src", "skills"));
|
||||
const transitionMatrix = TransitionMatrix.fromRegistry(skillRegistry);
|
||||
const agent = new Agent(natsClient, conversationStore, transitionMatrix);
|
||||
|
||||
// Register skills
|
||||
const shadowEvalBus = new ShadowEvalBus(garnetClient);
|
||||
const skillRegistry = SkillRegistry.load(resolve(process.cwd(), "src", "skills"));
|
||||
const factories: Record<string, SkillFactory> = {
|
||||
moderation: createModerationSkill,
|
||||
onboarding: createOnboardingSkill,
|
||||
supervisor: createSupervisorSkill,
|
||||
};
|
||||
const seenSkills = new Set<string>();
|
||||
for (const manifest of skillRegistry.list()) {
|
||||
@@ -260,12 +267,14 @@ async function bootstrap(): Promise<void> {
|
||||
}
|
||||
|
||||
installShutdownHooks(async () => {
|
||||
await stopMetricsServer();
|
||||
await natsClient.close();
|
||||
await garnetClient.close();
|
||||
await closeMongoDB();
|
||||
await shutdownTracing();
|
||||
});
|
||||
|
||||
startMetricsServer();
|
||||
await agent.start();
|
||||
|
||||
logger.info(`✓ ${serviceName} bootstrapped`);
|
||||
|
||||
@@ -3,10 +3,14 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import type { Logger } from 'pino';
|
||||
|
||||
import { getLogger } from './logger.js';
|
||||
import type { BudgetSnapshot } from '@agent/budget';
|
||||
|
||||
export interface RequestContext {
|
||||
chatroomId: string;
|
||||
deadlineAt?: string;
|
||||
budget?: BudgetSnapshot | undefined;
|
||||
handoffChain?: string[] | undefined;
|
||||
handoffDepth?: number | undefined;
|
||||
logger: Logger;
|
||||
requestId: string;
|
||||
skill: string;
|
||||
@@ -27,6 +31,7 @@ export function runWithContext<T>(
|
||||
chatroomId: context.chatroomId,
|
||||
requestId: context.requestId,
|
||||
skill: context.skill,
|
||||
handoffDepth: context.handoffDepth,
|
||||
trigger: context.trigger,
|
||||
userId: context.userId,
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ import { getLogLevel, getNodeEnv, getServiceName } from '../config.js';
|
||||
|
||||
type RequestBindings = {
|
||||
chatroomId?: string;
|
||||
handoffDepth?: number | undefined;
|
||||
requestId?: string;
|
||||
skill?: string;
|
||||
trigger?: string;
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { createServer, type Server } from "node:http";
|
||||
|
||||
import { getMetricsPort } from "../config.js";
|
||||
import { getLogger } from "./logger.js";
|
||||
import { renderMetrics } from "./metrics.js";
|
||||
|
||||
let server: Server | null = null;
|
||||
|
||||
export function startMetricsServer(port = getMetricsPort()): Server {
|
||||
if (server) {
|
||||
return server;
|
||||
}
|
||||
|
||||
server = createServer((request, response) => {
|
||||
if (request.url !== "/metrics") {
|
||||
response.writeHead(404);
|
||||
response.end("not found");
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "text/plain; version=0.0.4",
|
||||
});
|
||||
response.end(renderMetrics());
|
||||
});
|
||||
|
||||
server.listen(port, () => {
|
||||
getLogger().info({ port }, "✓ Metrics server listening");
|
||||
});
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
export async function stopMetricsServer(): Promise<void> {
|
||||
if (!server) {
|
||||
return;
|
||||
}
|
||||
const current = server;
|
||||
server = null;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
current.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { metrics, renderMetrics } from "./metrics.js";
|
||||
|
||||
describe("metrics", () => {
|
||||
it("renders the agent metric families", () => {
|
||||
const body = renderMetrics();
|
||||
|
||||
for (const name of [
|
||||
"agent_routing_accuracy_total",
|
||||
"agent_handoff_total",
|
||||
"agent_handoff_loop_total",
|
||||
"agent_handoff_count",
|
||||
"agent_unnecessary_escalation_total",
|
||||
"agent_missed_escalation_total",
|
||||
"agent_token_cost_usd",
|
||||
"agent_time_to_resolution_ms",
|
||||
"agent_validator_fail_total",
|
||||
"agent_human_override_total",
|
||||
]) {
|
||||
expect(body).toContain(`# HELP ${name}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("renders counter labels and histogram observations", () => {
|
||||
metrics.handoffTotal.inc({
|
||||
from: "onboarding",
|
||||
reason: "out-of-scope",
|
||||
to: "moderation",
|
||||
});
|
||||
metrics.handoffCount.observe(2);
|
||||
|
||||
const body = renderMetrics();
|
||||
|
||||
expect(body).toContain(
|
||||
'agent_handoff_total{from="onboarding",reason="out-of-scope",to="moderation"} 1',
|
||||
);
|
||||
expect(body).toContain('agent_handoff_count_bucket{le="2"} 1');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
type Labels = Record<string, string | number | undefined>;
|
||||
|
||||
interface MetricSample {
|
||||
labels: Labels;
|
||||
value: number;
|
||||
}
|
||||
|
||||
function labelKey(labels: Labels): string {
|
||||
return JSON.stringify(
|
||||
Object.fromEntries(
|
||||
Object.entries(labels)
|
||||
.filter(([, value]) => value !== undefined)
|
||||
.sort(([left], [right]) => left.localeCompare(right)),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function formatLabels(labels: Labels): string {
|
||||
const entries = Object.entries(labels).filter(([, value]) => value !== undefined);
|
||||
if (entries.length === 0) {
|
||||
return "";
|
||||
}
|
||||
return `{${entries
|
||||
.map(([key, value]) => `${key}="${String(value).replace(/"/gu, '\\"')}"`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
|
||||
class CounterMetric {
|
||||
private readonly samples = new Map<string, MetricSample>();
|
||||
|
||||
constructor(
|
||||
readonly name: string,
|
||||
readonly help: string,
|
||||
) {}
|
||||
|
||||
inc(labels: Labels = {}, value = 1): void {
|
||||
const key = labelKey(labels);
|
||||
const sample = this.samples.get(key) ?? { labels, value: 0 };
|
||||
sample.value += value;
|
||||
this.samples.set(key, sample);
|
||||
}
|
||||
|
||||
render(): string[] {
|
||||
return [
|
||||
`# HELP ${this.name} ${this.help}`,
|
||||
`# TYPE ${this.name} counter`,
|
||||
...[...this.samples.values()].map(
|
||||
(sample) => `${this.name}${formatLabels(sample.labels)} ${sample.value}`,
|
||||
),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
class HistogramMetric {
|
||||
private readonly values: number[] = [];
|
||||
|
||||
constructor(
|
||||
readonly name: string,
|
||||
readonly help: string,
|
||||
private readonly buckets: number[],
|
||||
) {}
|
||||
|
||||
observe(value: number): void {
|
||||
if (Number.isFinite(value)) {
|
||||
this.values.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
render(): string[] {
|
||||
const lines = [
|
||||
`# HELP ${this.name} ${this.help}`,
|
||||
`# TYPE ${this.name} histogram`,
|
||||
];
|
||||
for (const bucket of this.buckets) {
|
||||
const count = this.values.filter((value) => value <= bucket).length;
|
||||
lines.push(`${this.name}_bucket{le="${bucket}"} ${count}`);
|
||||
}
|
||||
lines.push(`${this.name}_bucket{le="+Inf"} ${this.values.length}`);
|
||||
lines.push(
|
||||
`${this.name}_sum ${this.values.reduce((sum, value) => sum + value, 0)}`,
|
||||
);
|
||||
lines.push(`${this.name}_count ${this.values.length}`);
|
||||
return lines;
|
||||
}
|
||||
}
|
||||
|
||||
export const metrics = {
|
||||
handoffTotal: new CounterMetric("agent_handoff_total", "Agent handoffs"),
|
||||
handoffLoopTotal: new CounterMetric(
|
||||
"agent_handoff_loop_total",
|
||||
"Agent handoff loops",
|
||||
),
|
||||
humanOverrideTotal: new CounterMetric(
|
||||
"agent_human_override_total",
|
||||
"Human override events",
|
||||
),
|
||||
missedEscalationTotal: new CounterMetric(
|
||||
"agent_missed_escalation_total",
|
||||
"Missed escalations",
|
||||
),
|
||||
routingAccuracyTotal: new CounterMetric(
|
||||
"agent_routing_accuracy_total",
|
||||
"Eval routing accuracy",
|
||||
),
|
||||
unnecessaryEscalationTotal: new CounterMetric(
|
||||
"agent_unnecessary_escalation_total",
|
||||
"Unnecessary escalations",
|
||||
),
|
||||
validatorFailTotal: new CounterMetric(
|
||||
"agent_validator_fail_total",
|
||||
"Validator failures",
|
||||
),
|
||||
handoffCount: new HistogramMetric(
|
||||
"agent_handoff_count",
|
||||
"Handoffs per request",
|
||||
[0, 1, 2, 3, 4],
|
||||
),
|
||||
timeToResolutionMs: new HistogramMetric(
|
||||
"agent_time_to_resolution_ms",
|
||||
"Time to resolution",
|
||||
[100, 500, 1_000, 2_500, 5_000, 10_000],
|
||||
),
|
||||
tokenCostUsd: new HistogramMetric(
|
||||
"agent_token_cost_usd",
|
||||
"Estimated LLM token cost USD",
|
||||
[0.001, 0.01, 0.05, 0.1, 0.5, 1],
|
||||
),
|
||||
};
|
||||
|
||||
export function renderMetrics(): string {
|
||||
return Object.values(metrics)
|
||||
.flatMap((metric) => metric.render())
|
||||
.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { parseControlTokens } from "./control-token-parser.js";
|
||||
|
||||
describe("parseControlTokens", () => {
|
||||
it("extracts handoff control tokens and strips them from text", () => {
|
||||
const parsed = parseControlTokens(
|
||||
"잠시 확인할게요. [HANDOFF:moderation:policy-risk]",
|
||||
);
|
||||
|
||||
expect(parsed.stripped).toBe("잠시 확인할게요.");
|
||||
expect(parsed.tokens).toEqual([
|
||||
{
|
||||
kind: "handoff",
|
||||
raw: "[HANDOFF:moderation:policy-risk]",
|
||||
reason: "policy-risk",
|
||||
targetAgent: "moderation",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts no-response tokens", () => {
|
||||
const parsed = parseControlTokens("[NO_RESPONSE]");
|
||||
|
||||
expect(parsed.stripped).toBe("");
|
||||
expect(parsed.tokens).toEqual([
|
||||
{
|
||||
kind: "no-response",
|
||||
raw: "[NO_RESPONSE]",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps invalid handoff reasons visible", () => {
|
||||
const parsed = parseControlTokens("[HANDOFF:moderation:not-a-reason]");
|
||||
|
||||
expect(parsed.stripped).toBe("[HANDOFF:moderation:not-a-reason]");
|
||||
expect(parsed.tokens).toEqual([]);
|
||||
});
|
||||
|
||||
it("accepts all handoff reason codes", () => {
|
||||
const reasons = [
|
||||
"out-of-scope",
|
||||
"missing-tool",
|
||||
"low-confidence",
|
||||
"validation-failed",
|
||||
"same-agent-revisit",
|
||||
"max-hop",
|
||||
"high-risk-action",
|
||||
"policy-risk",
|
||||
];
|
||||
|
||||
for (const reason of reasons) {
|
||||
const parsed = parseControlTokens(`[HANDOFF:supervisor:${reason}]`);
|
||||
expect(parsed.tokens[0]).toMatchObject({
|
||||
kind: "handoff",
|
||||
reason,
|
||||
targetAgent: "supervisor",
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
|
||||
const HANDOFF_REASONS = new Set<HandoffReason>([
|
||||
"out-of-scope",
|
||||
"missing-tool",
|
||||
"low-confidence",
|
||||
"validation-failed",
|
||||
"same-agent-revisit",
|
||||
"max-hop",
|
||||
"high-risk-action",
|
||||
"policy-risk",
|
||||
]);
|
||||
|
||||
export type ControlToken =
|
||||
| { kind: "no-response"; raw: string }
|
||||
| {
|
||||
kind: "handoff";
|
||||
raw: string;
|
||||
reason: HandoffReason;
|
||||
targetAgent: string;
|
||||
};
|
||||
|
||||
export function parseControlTokens(text: string): {
|
||||
stripped: string;
|
||||
tokens: ControlToken[];
|
||||
} {
|
||||
const tokens: ControlToken[] = [];
|
||||
let stripped = text;
|
||||
|
||||
stripped = stripped.replace(/\[NO_RESPONSE\]/gu, (raw) => {
|
||||
tokens.push({ kind: "no-response", raw });
|
||||
return "";
|
||||
});
|
||||
|
||||
stripped = stripped.replace(
|
||||
/\[HANDOFF:([a-z0-9_-]+):([a-z-]+)\]/giu,
|
||||
(raw, target: string, reason: string) => {
|
||||
if (!HANDOFF_REASONS.has(reason as HandoffReason)) {
|
||||
return raw;
|
||||
}
|
||||
tokens.push({
|
||||
kind: "handoff",
|
||||
raw,
|
||||
reason: reason as HandoffReason,
|
||||
targetAgent: target,
|
||||
});
|
||||
return "";
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
stripped: stripped.trim(),
|
||||
tokens,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import { decide } from "./trigger.js";
|
||||
|
||||
describe("escalation trigger", () => {
|
||||
it("turns explicit handoff reasons into structured requests", () => {
|
||||
const reasons: HandoffReason[] = [
|
||||
"out-of-scope",
|
||||
"missing-tool",
|
||||
"low-confidence",
|
||||
"validation-failed",
|
||||
"same-agent-revisit",
|
||||
"max-hop",
|
||||
"high-risk-action",
|
||||
"policy-risk",
|
||||
];
|
||||
|
||||
for (const reason of reasons) {
|
||||
const decision = decide({
|
||||
classified: { confidence: 0.8 },
|
||||
reason,
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "review",
|
||||
});
|
||||
|
||||
expect(decision).toMatchObject({
|
||||
reason,
|
||||
shouldHandoff: true,
|
||||
handoff: {
|
||||
reason,
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "review",
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("escalates exhausted budgets and validation rejects", () => {
|
||||
expect(
|
||||
decide({
|
||||
budget: {
|
||||
estimatedCostUsd: 0,
|
||||
handoffs: 4,
|
||||
maxCostUsd: 0.5,
|
||||
maxHandoffs: 4,
|
||||
maxSameAgentVisits: 2,
|
||||
maxToolCalls: 20,
|
||||
toolCalls: 0,
|
||||
visits: {},
|
||||
},
|
||||
}).handoff?.reason,
|
||||
).toBe("max-hop");
|
||||
|
||||
expect(decide({ validation: { status: "reject" } }).handoff?.reason).toBe(
|
||||
"validation-failed",
|
||||
);
|
||||
});
|
||||
|
||||
it("escalates low-confidence and uncertain high-risk responses", () => {
|
||||
expect(
|
||||
decide({ classified: { confidence: 0.2 } }).handoff?.reason,
|
||||
).toBe("low-confidence");
|
||||
|
||||
expect(
|
||||
decide({
|
||||
capabilityCard: {
|
||||
canHandle: ["policy"],
|
||||
cannotHandle: [],
|
||||
domain: "moderation",
|
||||
handoffTargets: ["supervisor"],
|
||||
outputContract: { format: "text", maxChars: "500" },
|
||||
riskTier: "high",
|
||||
},
|
||||
classified: { confidence: 0.6 },
|
||||
}).handoff?.reason,
|
||||
).toBe("high-risk-action");
|
||||
});
|
||||
|
||||
it("does not escalate ordinary confident responses", () => {
|
||||
expect(
|
||||
decide({
|
||||
capabilityCard: {
|
||||
canHandle: ["help"],
|
||||
cannotHandle: [],
|
||||
domain: "onboarding",
|
||||
handoffTargets: ["supervisor"],
|
||||
outputContract: { format: "text", maxChars: "500" },
|
||||
riskTier: "medium",
|
||||
},
|
||||
classified: { confidence: 0.9 },
|
||||
}),
|
||||
).toEqual({ shouldHandoff: false });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import type { HandoffReason, HandoffRequest } from "@agent/types";
|
||||
import type { BudgetSnapshot } from "@agent/budget";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
|
||||
export interface EscalationDecision {
|
||||
handoff?: HandoffRequest | undefined;
|
||||
reason?: HandoffReason | undefined;
|
||||
shouldHandoff: boolean;
|
||||
}
|
||||
|
||||
export function decide(input: {
|
||||
budget?: BudgetSnapshot | undefined;
|
||||
capabilityCard?: SkillCapabilityCard | undefined;
|
||||
classified?: { confidence?: number | undefined } | undefined;
|
||||
reason?: HandoffReason | undefined;
|
||||
targetAgent?: string | undefined;
|
||||
taskSpec?: string | undefined;
|
||||
validation?: { status?: string | undefined } | undefined;
|
||||
}): EscalationDecision {
|
||||
const confidence = input.classified?.confidence;
|
||||
let reason: HandoffReason | undefined = input.reason;
|
||||
|
||||
if (!reason) {
|
||||
if (input.budget && input.budget.handoffs >= input.budget.maxHandoffs) {
|
||||
reason = "max-hop";
|
||||
} else if (input.validation?.status === "reject") {
|
||||
reason = "validation-failed";
|
||||
} else if (typeof confidence === "number" && confidence < 0.35) {
|
||||
reason = "low-confidence";
|
||||
} else if (
|
||||
input.capabilityCard?.riskTier === "high" &&
|
||||
typeof confidence === "number" &&
|
||||
confidence < 0.7
|
||||
) {
|
||||
reason = "high-risk-action";
|
||||
}
|
||||
}
|
||||
|
||||
if (!reason) {
|
||||
return { shouldHandoff: false };
|
||||
}
|
||||
|
||||
return {
|
||||
handoff: {
|
||||
confidence: confidence ?? 0.5,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason,
|
||||
risk: input.capabilityCard?.riskTier ?? "medium",
|
||||
targetAgent: input.targetAgent ?? "supervisor",
|
||||
taskSpec: input.taskSpec ?? "Escalate this request to the appropriate skill.",
|
||||
},
|
||||
reason,
|
||||
shouldHandoff: true,
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,15 @@ name: moderation
|
||||
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
|
||||
version: 1
|
||||
priority: 50
|
||||
domain: moderation
|
||||
canHandle: [rules, violation-report, appeal, sanctions, policy-risk]
|
||||
cannotHandle: [onboarding-status, nickname-format, photo-verification, discord-onboarding]
|
||||
handoffTargets: [onboarding, supervisor]
|
||||
riskTier: high
|
||||
outputContract:
|
||||
format: text
|
||||
maxChars: "500"
|
||||
requiredFields: text
|
||||
intents: [greeting, rule-question, violation-report, appeal, off-topic]
|
||||
triggers: [user-message]
|
||||
tools: [lookup-rule]
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
||||
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
||||
@@ -22,6 +23,24 @@ type RuleTopic =
|
||||
| "sanctions"
|
||||
| "spam";
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: ["rules", "violation-report", "appeal", "sanctions", "policy-risk"],
|
||||
cannotHandle: [
|
||||
"onboarding-status",
|
||||
"nickname-format",
|
||||
"photo-verification",
|
||||
"discord-onboarding",
|
||||
],
|
||||
domain: "moderation",
|
||||
handoffTargets: ["onboarding", "supervisor"],
|
||||
outputContract: {
|
||||
format: "text",
|
||||
maxChars: "500",
|
||||
requiredFields: "text",
|
||||
},
|
||||
riskTier: "high",
|
||||
};
|
||||
|
||||
function formatHistory(history: ConversationMessage[]): string {
|
||||
if (history.length === 0) {
|
||||
return "(none)";
|
||||
@@ -156,6 +175,10 @@ export class ModerationSkill implements Skill {
|
||||
return request.skill === this.name;
|
||||
}
|
||||
|
||||
capabilityCard(): SkillCapabilityCard {
|
||||
return CAPABILITY_CARD;
|
||||
}
|
||||
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
|
||||
@@ -3,6 +3,15 @@ name: onboarding
|
||||
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
|
||||
version: 1
|
||||
priority: 70
|
||||
domain: onboarding
|
||||
canHandle: [new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed, onboarding-help]
|
||||
cannotHandle: [moderation-report, sanctions, appeal, policy-enforcement]
|
||||
handoffTargets: [moderation, supervisor]
|
||||
riskTier: medium
|
||||
outputContract:
|
||||
format: text
|
||||
maxChars: "500"
|
||||
requiredFields: text
|
||||
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
|
||||
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
|
||||
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
|
||||
|
||||
@@ -79,4 +79,14 @@ describe("validateOnboardingResponse", () => {
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
});
|
||||
|
||||
it("passes handoff control tokens", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"[HANDOFF:moderation:policy-risk]",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
expect(result.reasons).toContain("handoff");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { getMaxResponseLength } from "../../../config.js";
|
||||
import { parseControlTokens } from "../../control-token-parser.js";
|
||||
import { VALID_TIER_SET } from "../constants.js";
|
||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
@@ -131,15 +132,12 @@ export function validateOnboardingResponse(
|
||||
derived: DerivedState,
|
||||
options?: { allowShortResponse?: boolean | undefined },
|
||||
): ValidationResult {
|
||||
const text = postProcessResponse(draft);
|
||||
const parsedTokens = parseControlTokens(postProcessResponse(draft));
|
||||
const text = parsedTokens.stripped;
|
||||
const rejectReasons: string[] = [];
|
||||
const softWarnReasons: string[] = [];
|
||||
|
||||
if (!text) {
|
||||
rejectReasons.push("empty");
|
||||
}
|
||||
|
||||
if (text.includes("[NO_RESPONSE]")) {
|
||||
if (parsedTokens.tokens.some((token) => token.kind === "no-response")) {
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text: "[NO_RESPONSE]",
|
||||
@@ -147,6 +145,18 @@ export function validateOnboardingResponse(
|
||||
});
|
||||
}
|
||||
|
||||
if (parsedTokens.tokens.some((token) => token.kind === "handoff")) {
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text,
|
||||
reasons: ["handoff"],
|
||||
});
|
||||
}
|
||||
|
||||
if (!text) {
|
||||
rejectReasons.push("empty");
|
||||
}
|
||||
|
||||
if (!options?.allowShortResponse && text.length < 10) {
|
||||
rejectReasons.push("minLength");
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { getContext, getRequestLogger } from '@obs/context';
|
||||
import type { LlmClient } from '@llm/llm-client';
|
||||
import type { RoomContextService } from '@room/room-context-service';
|
||||
import type { Skill } from '@skills/skill';
|
||||
import type { SkillCapabilityCard } from '@skills/skill-loader';
|
||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||
import { createOnboardingToolRegistry } from './tools.js';
|
||||
@@ -68,6 +69,32 @@ import type {
|
||||
StoredOnboardingSession,
|
||||
} from './types.js';
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: [
|
||||
'new-member',
|
||||
'nickname-invalid',
|
||||
'photo-failed',
|
||||
'progress-update',
|
||||
'timeout-warning',
|
||||
'completed',
|
||||
'onboarding-help',
|
||||
],
|
||||
cannotHandle: [
|
||||
'moderation-report',
|
||||
'sanctions',
|
||||
'appeal',
|
||||
'policy-enforcement',
|
||||
],
|
||||
domain: 'onboarding',
|
||||
handoffTargets: ['moderation', 'supervisor'],
|
||||
outputContract: {
|
||||
format: 'text',
|
||||
maxChars: '500',
|
||||
requiredFields: 'text',
|
||||
},
|
||||
riskTier: 'medium',
|
||||
};
|
||||
|
||||
export class OnboardingSkill implements Skill {
|
||||
readonly name = 'onboarding';
|
||||
readonly priority = 70;
|
||||
@@ -99,6 +126,10 @@ export class OnboardingSkill implements Skill {
|
||||
return request.skill === 'onboarding';
|
||||
}
|
||||
|
||||
capabilityCard(): SkillCapabilityCard {
|
||||
return CAPABILITY_CARD;
|
||||
}
|
||||
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
conversationHistory: ConversationMessage[],
|
||||
|
||||
@@ -21,6 +21,15 @@ function makeTempSkillRoot(): string {
|
||||
"priority: 65",
|
||||
"intents: [hello, help]",
|
||||
"triggers: [user-message]",
|
||||
"domain: demo-domain",
|
||||
"canHandle: [demo-task]",
|
||||
"cannotHandle: [other-task]",
|
||||
"handoffTargets: [supervisor]",
|
||||
"riskTier: medium",
|
||||
"outputContract:",
|
||||
" format: text",
|
||||
" maxChars: \"500\"",
|
||||
" requiredFields: text",
|
||||
"tools: [demo-tool]",
|
||||
"resources:",
|
||||
" guide: ./resources/guide.md",
|
||||
@@ -41,8 +50,18 @@ describe("SkillRegistry", () => {
|
||||
expect(manifest).toMatchObject({
|
||||
name: "demo",
|
||||
description: "Demo skill.",
|
||||
canHandle: ["demo-task"],
|
||||
cannotHandle: ["other-task"],
|
||||
domain: "demo-domain",
|
||||
handoffTargets: ["supervisor"],
|
||||
intents: ["hello", "help"],
|
||||
outputContract: {
|
||||
format: "text",
|
||||
maxChars: "500",
|
||||
requiredFields: "text",
|
||||
},
|
||||
priority: 65,
|
||||
riskTier: "medium",
|
||||
tools: ["demo-tool"],
|
||||
trustTier: "T2",
|
||||
version: 1,
|
||||
|
||||
@@ -3,14 +3,36 @@ import { dirname, resolve } from "node:path";
|
||||
import { getLogger } from "@obs/logger";
|
||||
|
||||
export type TrustTier = "T1" | "T2" | "T3" | "T4";
|
||||
export type SkillRiskTier = "low" | "medium" | "high";
|
||||
|
||||
export interface SkillOutputContract {
|
||||
format?: string | undefined;
|
||||
maxChars?: string | undefined;
|
||||
requiredFields?: string | undefined;
|
||||
}
|
||||
|
||||
export interface SkillCapabilityCard {
|
||||
canHandle: string[];
|
||||
cannotHandle: string[];
|
||||
domain: string;
|
||||
handoffTargets: string[];
|
||||
outputContract: SkillOutputContract;
|
||||
riskTier: SkillRiskTier;
|
||||
}
|
||||
|
||||
export interface SkillManifest {
|
||||
bodyPath: string;
|
||||
canHandle: string[];
|
||||
cannotHandle: string[];
|
||||
description: string;
|
||||
domain: string;
|
||||
handoffTargets: string[];
|
||||
intents: string[];
|
||||
name: string;
|
||||
outputContract: SkillOutputContract;
|
||||
priority: number;
|
||||
resources: Record<string, string>;
|
||||
riskTier: SkillRiskTier;
|
||||
tools: string[];
|
||||
triggers: string[];
|
||||
trustTier: TrustTier;
|
||||
@@ -121,10 +143,32 @@ function readResources(
|
||||
);
|
||||
}
|
||||
|
||||
function readStringMap(
|
||||
fields: Record<string, unknown>,
|
||||
key: string,
|
||||
): Record<string, string> {
|
||||
const raw = fields[key];
|
||||
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(raw as Record<string, unknown>).flatMap(([mapKey, value]) =>
|
||||
typeof value === "string" && value.trim()
|
||||
? [[mapKey, value.trim().replace(/^["']|["']$/gu, "")]]
|
||||
: [],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function isTrustTier(value: unknown): value is TrustTier {
|
||||
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
|
||||
}
|
||||
|
||||
function isRiskTier(value: unknown): value is SkillRiskTier {
|
||||
return value === "low" || value === "medium" || value === "high";
|
||||
}
|
||||
|
||||
function isPriority(value: unknown): value is number {
|
||||
return (
|
||||
typeof value === "number" &&
|
||||
@@ -147,6 +191,8 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
||||
const priority = parsed.fields["priority"] ?? 50;
|
||||
const version = parsed.fields["version"];
|
||||
const trustTier = parsed.fields["trustTier"];
|
||||
const domain = parsed.fields["domain"];
|
||||
const riskTier = parsed.fields["riskTier"] ?? "low";
|
||||
|
||||
if (
|
||||
typeof name !== "string" ||
|
||||
@@ -155,20 +201,28 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
||||
!description.trim() ||
|
||||
!isPriority(priority) ||
|
||||
typeof version !== "number" ||
|
||||
!isTrustTier(trustTier)
|
||||
!isTrustTier(trustTier) ||
|
||||
!isRiskTier(riskTier)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const outputContract = readStringMap(parsed.fields, "outputContract");
|
||||
|
||||
return {
|
||||
body: parsed.body,
|
||||
manifest: {
|
||||
bodyPath: path,
|
||||
canHandle: readStringArray(parsed.fields, "canHandle"),
|
||||
cannotHandle: readStringArray(parsed.fields, "cannotHandle"),
|
||||
description,
|
||||
domain: typeof domain === "string" && domain.trim() ? domain : "general",
|
||||
handoffTargets: readStringArray(parsed.fields, "handoffTargets"),
|
||||
intents: readStringArray(parsed.fields, "intents"),
|
||||
name,
|
||||
outputContract,
|
||||
priority,
|
||||
resources: readResources(parsed.fields, skillDir),
|
||||
riskTier,
|
||||
tools: readStringArray(parsed.fields, "tools"),
|
||||
triggers: readStringArray(parsed.fields, "triggers"),
|
||||
trustTier,
|
||||
|
||||
@@ -3,10 +3,12 @@ import type {
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import type { SkillCapabilityCard } from "./skill-loader.js";
|
||||
|
||||
export interface Skill {
|
||||
readonly name: string;
|
||||
readonly priority: number;
|
||||
capabilityCard?(): SkillCapabilityCard;
|
||||
canHandle(request: AgentRequest): boolean;
|
||||
handle(
|
||||
request: AgentRequest,
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
---
|
||||
name: supervisor
|
||||
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
|
||||
version: 1
|
||||
priority: 10
|
||||
domain: meta
|
||||
canHandle: [*]
|
||||
cannotHandle: []
|
||||
handoffTargets: [onboarding, moderation]
|
||||
riskTier: medium
|
||||
outputContract:
|
||||
format: text
|
||||
maxChars: "500"
|
||||
requiredFields: text
|
||||
intents: []
|
||||
triggers: [handoff, user-message]
|
||||
tools: []
|
||||
resources:
|
||||
trustTier: T4
|
||||
---
|
||||
|
||||
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||
@@ -0,0 +1,9 @@
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillDeps } from "@skills/skill-factory";
|
||||
import { SupervisorSkill } from "./skill.js";
|
||||
|
||||
export { SupervisorSkill } from "./skill.js";
|
||||
|
||||
export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export function getIntentKinds(): string[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
export function getToolNames(): string[] {
|
||||
return [];
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
|
||||
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
|
||||
|
||||
export async function appendSupervisorReview(
|
||||
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
|
||||
...fields,
|
||||
type: "supervisor-handoff",
|
||||
queuedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
|
||||
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
|
||||
const text = [
|
||||
packet?.userGoal,
|
||||
packet?.summary,
|
||||
packet?.handoffReason,
|
||||
].join(" ");
|
||||
|
||||
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) {
|
||||
return "moderation";
|
||||
}
|
||||
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) {
|
||||
return "onboarding";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import { SupervisorSkill } from "./skill.js";
|
||||
|
||||
class FakeGarnet {
|
||||
readonly streams: Array<{ key: string; fields: Record<string, unknown> }> = [];
|
||||
|
||||
appendStream(key: string, fields: Record<string, unknown>): Promise<string> {
|
||||
this.streams.push({ key, fields });
|
||||
return Promise.resolve("1-0");
|
||||
}
|
||||
|
||||
setStringIfAbsent(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
}
|
||||
|
||||
class FakeNats {
|
||||
readonly published: AlertPayload[] = [];
|
||||
|
||||
publish<T = unknown>(_subject: string, payload: T): Promise<void> {
|
||||
this.published.push(payload as AlertPayload);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
const baseRequest: AgentRequest = {
|
||||
chatroomId: "room-1",
|
||||
nickname: "tester",
|
||||
requestId: "request-1",
|
||||
skill: "supervisor",
|
||||
trigger: "handoff",
|
||||
userId: "user-1",
|
||||
};
|
||||
|
||||
const sourceResponse: AgentResponse = {
|
||||
confidence: 0.3,
|
||||
requestId: "request-1",
|
||||
source: "template",
|
||||
text: "needs review",
|
||||
};
|
||||
|
||||
describe("SupervisorSkill", () => {
|
||||
it("queues review and publishes alert for handoffs", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const skill = new SupervisorSkill(garnet, nats);
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
confidence: 0.3,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "max-hop",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "가입 상태 확인",
|
||||
},
|
||||
request: baseRequest,
|
||||
response: sourceResponse,
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
});
|
||||
|
||||
const response = await skill.handle(
|
||||
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||
[],
|
||||
);
|
||||
|
||||
expect(response.text).toContain("운영자");
|
||||
expect(response.metadata).toMatchObject({
|
||||
supervisorReason: "max-hop",
|
||||
suggestedTarget: "onboarding",
|
||||
});
|
||||
expect(garnet.streams[0]).toMatchObject({
|
||||
key: "lolympus-agent:review-queue",
|
||||
fields: {
|
||||
reason: "max-hop",
|
||||
requestId: "request-1",
|
||||
target: "onboarding",
|
||||
type: "supervisor-handoff",
|
||||
},
|
||||
});
|
||||
expect(nats.published[0]).toMatchObject({
|
||||
kind: "supervisor-handoff",
|
||||
metric: "agent.supervisor_handoff",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import { AlertService } from "@obs/alert";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import { appendSupervisorReview } from "./review-queue.js";
|
||||
import { routeSupervisorTarget } from "./routing-rules.js";
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: ["*"],
|
||||
cannotHandle: [],
|
||||
domain: "meta",
|
||||
handoffTargets: ["onboarding", "moderation"],
|
||||
outputContract: {
|
||||
format: "text",
|
||||
maxChars: "500",
|
||||
requiredFields: "text",
|
||||
},
|
||||
riskTier: "medium",
|
||||
};
|
||||
|
||||
export class SupervisorSkill implements Skill {
|
||||
readonly name = "supervisor";
|
||||
readonly priority = 10;
|
||||
private readonly alertService: AlertService;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
) {
|
||||
this.alertService = new AlertService(garnetClient, natsClient);
|
||||
}
|
||||
|
||||
canHandle(request: AgentRequest): boolean {
|
||||
return request.skill === this.name;
|
||||
}
|
||||
|
||||
capabilityCard(): SkillCapabilityCard {
|
||||
return CAPABILITY_CARD;
|
||||
}
|
||||
|
||||
async handle(
|
||||
request: AgentRequest,
|
||||
_conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const packet = readHandoffPacket(request);
|
||||
const target = routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
chatroomId: request.chatroomId,
|
||||
reason,
|
||||
requestId: request.requestId,
|
||||
target: target ?? "human-review",
|
||||
userId: request.userId,
|
||||
});
|
||||
metrics.humanOverrideTotal.inc({ kind: reason });
|
||||
|
||||
await this.alertService.sendAlert(
|
||||
{
|
||||
kind: "supervisor-handoff",
|
||||
metric: "agent.supervisor_handoff",
|
||||
severity: "warn",
|
||||
suggestedAction: "Review the supervisor handoff queue.",
|
||||
threshold: 0,
|
||||
timestamp: new Date().toISOString(),
|
||||
value: 1,
|
||||
window: "request",
|
||||
},
|
||||
{
|
||||
dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
supervisorReason: reason,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { SkillRegistry } from "./skill-loader.js";
|
||||
import { TransitionMatrix } from "./transition-matrix.js";
|
||||
|
||||
function writeSkill(
|
||||
root: string,
|
||||
name: string,
|
||||
handoffTargets: string[],
|
||||
): void {
|
||||
const skillDir = join(root, name);
|
||||
mkdirSync(skillDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(skillDir, "SKILL.md"),
|
||||
[
|
||||
"---",
|
||||
`name: ${name}`,
|
||||
`description: ${name} skill.`,
|
||||
"version: 1",
|
||||
"priority: 50",
|
||||
"domain: test",
|
||||
"canHandle: [test]",
|
||||
"cannotHandle: []",
|
||||
`handoffTargets: [${handoffTargets.join(", ")}]`,
|
||||
"riskTier: low",
|
||||
"outputContract:",
|
||||
" format: text",
|
||||
" maxChars: \"500\"",
|
||||
"trustTier: T2",
|
||||
"---",
|
||||
"",
|
||||
"body",
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
describe("TransitionMatrix", () => {
|
||||
it("allows declared handoffs and blocks self loops", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-"));
|
||||
writeSkill(root, "onboarding", ["moderation"]);
|
||||
writeSkill(root, "moderation", []);
|
||||
|
||||
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
|
||||
|
||||
expect(matrix.isAllowed("onboarding", "moderation")).toBe(true);
|
||||
expect(matrix.isAllowed("onboarding", "supervisor")).toBe(true);
|
||||
expect(matrix.isAllowed("onboarding", "onboarding")).toBe(false);
|
||||
expect(matrix.isAllowed("moderation", "onboarding")).toBe(false);
|
||||
});
|
||||
|
||||
it("reports unknown handoff targets", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-bad-"));
|
||||
writeSkill(root, "onboarding", ["missing"]);
|
||||
|
||||
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
|
||||
|
||||
expect(matrix.validateGraph(["onboarding"])).toContain(
|
||||
"onboarding: unknown handoff target missing",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { SkillRegistry } from "./skill-loader.js";
|
||||
|
||||
export class TransitionMatrix {
|
||||
private constructor(private readonly transitions: Map<string, Set<string>>) {}
|
||||
|
||||
static fromRegistry(registry: SkillRegistry): TransitionMatrix {
|
||||
const transitions = new Map<string, Set<string>>();
|
||||
for (const manifest of registry.list()) {
|
||||
const targets =
|
||||
manifest.name === "supervisor"
|
||||
? manifest.handoffTargets
|
||||
: [...manifest.handoffTargets, "supervisor"];
|
||||
transitions.set(
|
||||
manifest.name,
|
||||
new Set(targets),
|
||||
);
|
||||
}
|
||||
return new TransitionMatrix(transitions);
|
||||
}
|
||||
|
||||
isAllowed(from: string, to: string): boolean {
|
||||
if (from === to) {
|
||||
return false;
|
||||
}
|
||||
return this.transitions.get(from)?.has(to) ?? true;
|
||||
}
|
||||
|
||||
targetsOf(from: string): string[] {
|
||||
return [...(this.transitions.get(from) ?? new Set<string>())];
|
||||
}
|
||||
|
||||
validateGraph(knownSkills?: readonly string[]): string[] {
|
||||
const known = new Set([...(knownSkills ?? this.transitions.keys()), "supervisor"]);
|
||||
const errors: string[] = [];
|
||||
for (const [from, targets] of this.transitions.entries()) {
|
||||
if (targets.has(from)) {
|
||||
errors.push(`${from}: handoffTargets must not include itself`);
|
||||
}
|
||||
for (const target of targets) {
|
||||
if (!known.has(target)) {
|
||||
errors.push(`${from}: unknown handoff target ${target}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,10 @@ import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
|
||||
import { TransitionMatrix } from "./transition-matrix.js";
|
||||
import * as moderationBinding from "./moderation/manifest-binding.js";
|
||||
import * as onboardingBinding from "./onboarding/manifest-binding.js";
|
||||
import * as supervisorBinding from "./supervisor/manifest-binding.js";
|
||||
|
||||
interface ManifestBinding {
|
||||
getIntentKinds(): string[];
|
||||
@@ -14,6 +16,7 @@ interface ManifestBinding {
|
||||
const BINDINGS: Record<string, ManifestBinding> = {
|
||||
moderation: moderationBinding,
|
||||
onboarding: onboardingBinding,
|
||||
supervisor: supervisorBinding,
|
||||
};
|
||||
|
||||
function sortUnique(values: readonly string[]): string[] {
|
||||
@@ -32,12 +35,15 @@ function diff(left: readonly string[], right: readonly string[]): string[] {
|
||||
function validateManifest(
|
||||
manifest: SkillManifest,
|
||||
binding: ManifestBinding | undefined,
|
||||
knownSkills: readonly string[],
|
||||
): string[] {
|
||||
if (!binding) {
|
||||
return [`${manifest.name}: missing manifest binding`];
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
const capabilityErrors = validateCapabilityCard(manifest, knownSkills);
|
||||
errors.push(...capabilityErrors);
|
||||
const manifestIntents = sortUnique(manifest.intents);
|
||||
const codeIntents = sortUnique(binding.getIntentKinds());
|
||||
const unknownIntents = diff(manifestIntents, codeIntents);
|
||||
@@ -66,6 +72,45 @@ function validateManifest(
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateCapabilityCard(
|
||||
manifest: SkillManifest,
|
||||
knownSkills: readonly string[],
|
||||
): string[] {
|
||||
const errors: string[] = [];
|
||||
const canHandle = new Set(manifest.canHandle);
|
||||
const overlap = manifest.cannotHandle.filter((item) => canHandle.has(item));
|
||||
if (overlap.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: canHandle and cannotHandle overlap: ${format(overlap)}`,
|
||||
);
|
||||
}
|
||||
const riskTier: string = manifest.riskTier;
|
||||
if (riskTier !== "low" && riskTier !== "medium" && riskTier !== "high") {
|
||||
errors.push(`${manifest.name}: invalid riskTier ${riskTier}`);
|
||||
}
|
||||
const knownTargets = new Set([...knownSkills, "supervisor"]);
|
||||
const missingTargets = manifest.handoffTargets.filter(
|
||||
(target) => !knownTargets.has(target),
|
||||
);
|
||||
if (missingTargets.length > 0) {
|
||||
errors.push(
|
||||
`${manifest.name}: unknown handoffTargets: ${format(missingTargets)}`,
|
||||
);
|
||||
}
|
||||
if (manifest.handoffTargets.includes(manifest.name)) {
|
||||
errors.push(`${manifest.name}: handoffTargets must not include itself`);
|
||||
}
|
||||
if (!manifest.domain.trim()) {
|
||||
errors.push(`${manifest.name}: domain is required`);
|
||||
}
|
||||
if (!manifest.outputContract.format || !manifest.outputContract.maxChars) {
|
||||
errors.push(
|
||||
`${manifest.name}: outputContract.format and outputContract.maxChars are required`,
|
||||
);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||
const byPriority = new Map<number, string[]>();
|
||||
for (const manifest of manifests) {
|
||||
@@ -84,11 +129,13 @@ function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
|
||||
const registry = SkillRegistry.load(rootDir);
|
||||
const manifests = registry.list();
|
||||
const knownSkills = manifests.map((manifest) => manifest.name);
|
||||
return [
|
||||
...manifests.flatMap((manifest) =>
|
||||
validateManifest(manifest, BINDINGS[manifest.name]),
|
||||
validateManifest(manifest, BINDINGS[manifest.name], knownSkills),
|
||||
),
|
||||
...validatePriorityConflicts(manifests),
|
||||
...TransitionMatrix.fromRegistry(registry).validateGraph(knownSkills),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user