Add bounded skill handoff graph
This commit is contained in:
1 parent
54bc295ead
commit
c14adac162
42 files changed
+2243
-31
No files matched your search
+249
-1
@@ -1,7 +1,10 @@
|
|||||||
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
|
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
import { selectSkillForRequest } from "./agent.js";
|
import type { ConversationStore } from "@conv/conversation-store";
|
||||||
|
import type { NatsClient } from "@common/nats";
|
||||||
|
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||||
|
import { Agent, selectSkillForRequest } from "./agent.js";
|
||||||
|
|
||||||
function makeRequest(skill = "shared"): AgentRequest {
|
function makeRequest(skill = "shared"): AgentRequest {
|
||||||
return {
|
return {
|
||||||
@@ -32,6 +35,60 @@ function makeSkill(name: string, priority: number, canHandle = true): Skill {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class MemoryConversationStore {
|
||||||
|
readonly messages: ConversationMessage[] = [];
|
||||||
|
|
||||||
|
getHistory(): Promise<ConversationMessage[]> {
|
||||||
|
return Promise.resolve([...this.messages]);
|
||||||
|
}
|
||||||
|
|
||||||
|
appendMessage(
|
||||||
|
_chatroomId: string,
|
||||||
|
_userId: string,
|
||||||
|
message: ConversationMessage,
|
||||||
|
): Promise<void> {
|
||||||
|
this.messages.push(message);
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeNatsClient {
|
||||||
|
readonly responses: AgentResponse[] = [];
|
||||||
|
|
||||||
|
respond(_msg: unknown, response: AgentResponse): Promise<void> {
|
||||||
|
this.responses.push(response);
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleThroughAgent(
|
||||||
|
skills: readonly Skill[],
|
||||||
|
request: AgentRequest,
|
||||||
|
transitionMatrix?: TransitionMatrix,
|
||||||
|
): Promise<{
|
||||||
|
nats: FakeNatsClient;
|
||||||
|
store: MemoryConversationStore;
|
||||||
|
}> {
|
||||||
|
const nats = new FakeNatsClient();
|
||||||
|
const store = new MemoryConversationStore();
|
||||||
|
const agent = new Agent(
|
||||||
|
nats as unknown as NatsClient,
|
||||||
|
store as unknown as ConversationStore,
|
||||||
|
transitionMatrix,
|
||||||
|
);
|
||||||
|
for (const skill of skills) {
|
||||||
|
agent.registerSkill(skill);
|
||||||
|
}
|
||||||
|
|
||||||
|
await (agent as unknown as {
|
||||||
|
handleRequest(msg: { data: Uint8Array }): Promise<void>;
|
||||||
|
}).handleRequest({
|
||||||
|
data: new TextEncoder().encode(JSON.stringify(request)),
|
||||||
|
});
|
||||||
|
|
||||||
|
return { nats, store };
|
||||||
|
}
|
||||||
|
|
||||||
describe("selectSkillForRequest", () => {
|
describe("selectSkillForRequest", () => {
|
||||||
it("selects the highest-priority matching skill", () => {
|
it("selects the highest-priority matching skill", () => {
|
||||||
const request = makeRequest();
|
const request = makeRequest();
|
||||||
@@ -51,3 +108,194 @@ describe("selectSkillForRequest", () => {
|
|||||||
expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull();
|
expect(selectSkillForRequest([makeSkill("miss", 50, false)], makeRequest())).toBeNull();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("Agent handoff dispatch", () => {
|
||||||
|
it("responds once with the final handoff target response", async () => {
|
||||||
|
const source: Skill = {
|
||||||
|
...makeSkill("source", 50),
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 0.8,
|
||||||
|
metadata: {
|
||||||
|
handoffRequest: {
|
||||||
|
confidence: 0.8,
|
||||||
|
evidenceRefs: ["rule-1"],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "out-of-scope",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "target",
|
||||||
|
taskSpec: "continue",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "handoff",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const target: Skill = {
|
||||||
|
...makeSkill("target", 60),
|
||||||
|
canHandle: (request) => request.skill === "target",
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 1,
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "final answer",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const { nats, store } = await handleThroughAgent(
|
||||||
|
[source, target],
|
||||||
|
makeRequest("source"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(nats.responses).toHaveLength(1);
|
||||||
|
expect(nats.responses[0]?.text).toBe("final answer");
|
||||||
|
expect(store.messages.map((message) => message.content)).toEqual([
|
||||||
|
"final answer",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes handoff control tokens through metadata", async () => {
|
||||||
|
const source: Skill = {
|
||||||
|
...makeSkill("source", 50),
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 0.9,
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "[HANDOFF:target:missing-tool]",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const target: Skill = {
|
||||||
|
...makeSkill("target", 60),
|
||||||
|
canHandle: (request) => request.skill === "target",
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 1,
|
||||||
|
metadata: {
|
||||||
|
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "handled elsewhere",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const { nats } = await handleThroughAgent(
|
||||||
|
[source, target],
|
||||||
|
makeRequest("source"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(nats.responses[0]?.text).toBe("handled elsewhere");
|
||||||
|
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes runtime verifier escalations to supervisor", async () => {
|
||||||
|
const previous = process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
|
||||||
|
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = "true";
|
||||||
|
try {
|
||||||
|
const moderation: Skill = {
|
||||||
|
...makeSkill("moderation", 50),
|
||||||
|
canHandle: (request) => request.skill === "moderation",
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 0.9,
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "https://not-allowed.example",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const supervisor: Skill = {
|
||||||
|
...makeSkill("supervisor", 90),
|
||||||
|
canHandle: (request) => request.skill === "supervisor",
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 0.6,
|
||||||
|
metadata: {
|
||||||
|
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "sent to review",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const { nats } = await handleThroughAgent(
|
||||||
|
[moderation, supervisor],
|
||||||
|
makeRequest("moderation"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(nats.responses[0]?.text).toBe("sent to review");
|
||||||
|
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||||
|
} finally {
|
||||||
|
if (previous === undefined) {
|
||||||
|
delete process.env["AGENT_RUNTIME_VERIFIER_ENABLED"];
|
||||||
|
} else {
|
||||||
|
process.env["AGENT_RUNTIME_VERIFIER_ENABLED"] = previous;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reroutes illegal transitions to supervisor", async () => {
|
||||||
|
const source: Skill = {
|
||||||
|
...makeSkill("source", 50),
|
||||||
|
canHandle: (request) => request.skill === "source",
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 0.9,
|
||||||
|
metadata: {
|
||||||
|
handoffRequest: {
|
||||||
|
confidence: 0.9,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "out-of-scope",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "target",
|
||||||
|
taskSpec: "continue",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "handoff",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const target: Skill = {
|
||||||
|
...makeSkill("target", 60),
|
||||||
|
canHandle: (request) => request.skill === "target",
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 1,
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "should not run",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const supervisor: Skill = {
|
||||||
|
...makeSkill("supervisor", 10),
|
||||||
|
canHandle: (request) => request.skill === "supervisor",
|
||||||
|
handle: (request) =>
|
||||||
|
Promise.resolve({
|
||||||
|
confidence: 0.6,
|
||||||
|
metadata: {
|
||||||
|
packetSeen: Boolean(request.context?.["handoffPacket"]),
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "supervisor review",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const matrix = {
|
||||||
|
isAllowed: () => false,
|
||||||
|
} as unknown as TransitionMatrix;
|
||||||
|
const { nats } = await handleThroughAgent(
|
||||||
|
[source, target, supervisor],
|
||||||
|
makeRequest("source"),
|
||||||
|
matrix,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(nats.responses[0]?.text).toBe("supervisor review");
|
||||||
|
expect(nats.responses[0]?.metadata?.["packetSeen"]).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
+447
-19
@@ -5,8 +5,22 @@ import { getRequestLogger, runWithContext } from "@obs/context";
|
|||||||
import { getLogger } from "@obs/logger";
|
import { getLogger } from "@obs/logger";
|
||||||
import { withSkillSpan, withSpan } from "@obs/spans";
|
import { withSkillSpan, withSpan } from "@obs/spans";
|
||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
import type { AgentRequest, AgentResponse } from "./types.js";
|
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||||
|
import { parseControlTokens } from "@skills/control-token-parser";
|
||||||
|
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||||
|
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
|
||||||
|
import { metrics } from "@obs/metrics";
|
||||||
|
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
|
||||||
|
import { buildHandoffPacket } from "./handoff-packet.js";
|
||||||
|
import type {
|
||||||
|
AgentRequest,
|
||||||
|
AgentResponse,
|
||||||
|
ConversationMessage,
|
||||||
|
HandoffReason,
|
||||||
|
HandoffRequest,
|
||||||
|
} from "./types.js";
|
||||||
import {
|
import {
|
||||||
|
getAgentRuntimeVerifierEnabled,
|
||||||
getAgentRequestTimeoutMs,
|
getAgentRequestTimeoutMs,
|
||||||
getRequestSubject,
|
getRequestSubject,
|
||||||
getSafetyMarginMs,
|
getSafetyMarginMs,
|
||||||
@@ -43,6 +57,7 @@ export class Agent {
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly natsClient: NatsClient,
|
private readonly natsClient: NatsClient,
|
||||||
private readonly conversationStore: ConversationStore,
|
private readonly conversationStore: ConversationStore,
|
||||||
|
private readonly transitionMatrix?: TransitionMatrix | undefined,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
registerSkill(skill: Skill): void {
|
registerSkill(skill: Skill): void {
|
||||||
@@ -107,18 +122,28 @@ export class Agent {
|
|||||||
"[Agent] Request received",
|
"[Agent] Request received",
|
||||||
);
|
);
|
||||||
|
|
||||||
const skill = selectSkillForRequest(this.skills, request);
|
let skill = selectSkillForRequest(this.skills, request);
|
||||||
if (!skill) {
|
if (!skill) {
|
||||||
logger.info("[Agent] No skill found for request");
|
const supervisorRequest: AgentRequest = {
|
||||||
const response: AgentResponse = {
|
...request,
|
||||||
requestId: request.requestId,
|
skill: "supervisor",
|
||||||
text: "",
|
trigger: "handoff",
|
||||||
confidence: 0,
|
|
||||||
source: "template",
|
|
||||||
};
|
};
|
||||||
rootSpan.setAttribute("agent.source", "template");
|
skill = selectSkillForRequest(this.skills, supervisorRequest);
|
||||||
await this.natsClient.respond(msg, response);
|
if (skill) {
|
||||||
return;
|
request = supervisorRequest;
|
||||||
|
} else {
|
||||||
|
logger.info("[Agent] No skill found for request");
|
||||||
|
const response: AgentResponse = {
|
||||||
|
requestId: request.requestId,
|
||||||
|
text: "",
|
||||||
|
confidence: 0,
|
||||||
|
source: "template",
|
||||||
|
};
|
||||||
|
rootSpan.setAttribute("agent.source", "template");
|
||||||
|
await this.natsClient.respond(msg, response);
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -139,14 +164,17 @@ export class Agent {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await withSkillSpan(
|
const startedAt = Date.now();
|
||||||
skill.name,
|
const budget = new BudgetTracker();
|
||||||
{
|
budget.recordVisit(skill.name);
|
||||||
"agent.skill": skill.name,
|
const response = await this.dispatch({
|
||||||
"agent.trigger": request.trigger,
|
budget,
|
||||||
},
|
chain: [skill.name],
|
||||||
() => skill.handle(request, history),
|
depth: 0,
|
||||||
);
|
history,
|
||||||
|
request,
|
||||||
|
skill,
|
||||||
|
});
|
||||||
|
|
||||||
if (response.text.trim()) {
|
if (response.text.trim()) {
|
||||||
await this.conversationStore.appendMessage(
|
await this.conversationStore.appendMessage(
|
||||||
@@ -161,6 +189,8 @@ export class Agent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
await this.natsClient.respond(msg, response);
|
await this.natsClient.respond(msg, response);
|
||||||
|
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
|
||||||
|
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
|
||||||
rootSpan.setAttribute(
|
rootSpan.setAttribute(
|
||||||
"agent.source",
|
"agent.source",
|
||||||
response.text.trim() ? response.source : "empty",
|
response.text.trim() ? response.source : "empty",
|
||||||
@@ -200,4 +230,402 @@ export class Agent {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async dispatch(input: {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
chain: string[];
|
||||||
|
depth: number;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
request: AgentRequest;
|
||||||
|
skill: Skill;
|
||||||
|
}): Promise<AgentResponse> {
|
||||||
|
const { budget, chain, depth, history, request, skill } = input;
|
||||||
|
const contextBase = {
|
||||||
|
chatroomId: request.chatroomId,
|
||||||
|
deadlineAt: new Date(
|
||||||
|
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
||||||
|
).toISOString(),
|
||||||
|
requestId: request.requestId,
|
||||||
|
skill: skill.name,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
trigger: request.trigger,
|
||||||
|
userId: request.userId,
|
||||||
|
};
|
||||||
|
|
||||||
|
return runWithContext(
|
||||||
|
{
|
||||||
|
...contextBase,
|
||||||
|
budget: budget.snapshot(),
|
||||||
|
handoffChain: chain,
|
||||||
|
handoffDepth: depth,
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
const response = await this.runSkillOnce({
|
||||||
|
chain,
|
||||||
|
depth,
|
||||||
|
history,
|
||||||
|
request,
|
||||||
|
skill,
|
||||||
|
});
|
||||||
|
const normalized = this.normalizeControlTokens(response, skill.name);
|
||||||
|
const verified = await this.applyRuntimeVerifier({
|
||||||
|
chain,
|
||||||
|
depth,
|
||||||
|
history,
|
||||||
|
request,
|
||||||
|
response: normalized,
|
||||||
|
skill,
|
||||||
|
});
|
||||||
|
const escalated = this.applyEscalationTrigger({
|
||||||
|
budget,
|
||||||
|
request,
|
||||||
|
response: verified,
|
||||||
|
skill,
|
||||||
|
});
|
||||||
|
|
||||||
|
const handoff = readHandoffRequest(escalated);
|
||||||
|
if (!handoff) {
|
||||||
|
return escalated;
|
||||||
|
}
|
||||||
|
|
||||||
|
const next = this.prepareHandoff({
|
||||||
|
budget,
|
||||||
|
chain,
|
||||||
|
currentSkill: skill.name,
|
||||||
|
depth,
|
||||||
|
handoff,
|
||||||
|
history,
|
||||||
|
request,
|
||||||
|
response: escalated,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!next) {
|
||||||
|
return this.makeSupervisorFallback(request, handoff.reason);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.dispatch(next);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runSkillOnce(input: {
|
||||||
|
chain: string[];
|
||||||
|
depth: number;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
request: AgentRequest;
|
||||||
|
skill: Skill;
|
||||||
|
}): Promise<AgentResponse> {
|
||||||
|
return withSkillSpan(
|
||||||
|
input.skill.name,
|
||||||
|
{
|
||||||
|
"agent.handoff_chain": input.chain.join(">"),
|
||||||
|
"agent.handoff_depth": input.depth,
|
||||||
|
"agent.skill": input.skill.name,
|
||||||
|
"agent.trigger": input.request.trigger,
|
||||||
|
},
|
||||||
|
() => input.skill.handle(input.request, input.history),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private normalizeControlTokens(
|
||||||
|
response: AgentResponse,
|
||||||
|
currentSkill: string,
|
||||||
|
): AgentResponse {
|
||||||
|
const parsed = parseControlTokens(response.text);
|
||||||
|
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||||
|
const metadata = { ...(response.metadata ?? {}) };
|
||||||
|
if (handoff?.kind === "handoff") {
|
||||||
|
metadata.handoffRequest = {
|
||||||
|
confidence: response.confidence,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: handoff.reason,
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: handoff.targetAgent,
|
||||||
|
taskSpec: `Continue handling from ${currentSkill}`,
|
||||||
|
} satisfies HandoffRequest;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...response,
|
||||||
|
metadata,
|
||||||
|
text: parsed.stripped,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async applyRuntimeVerifier(input: {
|
||||||
|
chain: string[];
|
||||||
|
depth: number;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
skill: Skill;
|
||||||
|
}): Promise<AgentResponse> {
|
||||||
|
if (!getAgentRuntimeVerifierEnabled()) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
|
||||||
|
if (!verifier) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
const verdict = await verifier({
|
||||||
|
request: input.request,
|
||||||
|
response: input.response,
|
||||||
|
skill: input.skill.name,
|
||||||
|
});
|
||||||
|
if (verdict === "pass") {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
|
||||||
|
if (verdict === "retry" && input.depth === 0) {
|
||||||
|
const retryResponse = await this.runSkillOnce({
|
||||||
|
...input,
|
||||||
|
request: {
|
||||||
|
...input.request,
|
||||||
|
context: {
|
||||||
|
...(input.request.context ?? {}),
|
||||||
|
runtimeVerifierRetry: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const normalizedRetry = this.normalizeControlTokens(
|
||||||
|
retryResponse,
|
||||||
|
input.skill.name,
|
||||||
|
);
|
||||||
|
if (readHandoffRequest(normalizedRetry)) {
|
||||||
|
return normalizedRetry;
|
||||||
|
}
|
||||||
|
const retryVerdict = await verifier({
|
||||||
|
request: input.request,
|
||||||
|
response: normalizedRetry,
|
||||||
|
skill: input.skill.name,
|
||||||
|
});
|
||||||
|
if (retryVerdict === "pass") {
|
||||||
|
return {
|
||||||
|
...normalizedRetry,
|
||||||
|
metadata: {
|
||||||
|
...(normalizedRetry.metadata ?? {}),
|
||||||
|
runtimeVerifier: "retry-pass",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
metrics.validatorFailTotal.inc({
|
||||||
|
skill: input.skill.name,
|
||||||
|
stage: `retry-${retryVerdict}`,
|
||||||
|
});
|
||||||
|
return this.withRuntimeVerifierHandoff(normalizedRetry);
|
||||||
|
}
|
||||||
|
return this.withRuntimeVerifierHandoff(input.response);
|
||||||
|
}
|
||||||
|
|
||||||
|
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
||||||
|
return {
|
||||||
|
...response,
|
||||||
|
metadata: {
|
||||||
|
...(response.metadata ?? {}),
|
||||||
|
handoffRequest: {
|
||||||
|
confidence: response.confidence,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "validation-failed",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "Runtime verifier rejected the response.",
|
||||||
|
} satisfies HandoffRequest,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private applyEscalationTrigger(input: {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
skill: Skill;
|
||||||
|
}): AgentResponse {
|
||||||
|
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
const decision = decideEscalation({
|
||||||
|
budget: input.budget.snapshot(),
|
||||||
|
capabilityCard: input.skill.capabilityCard?.(),
|
||||||
|
classified: { confidence: input.response.confidence },
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
||||||
|
});
|
||||||
|
if (!decision.handoff) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
...input.response,
|
||||||
|
metadata: {
|
||||||
|
...(input.response.metadata ?? {}),
|
||||||
|
handoffRequest: decision.handoff,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private prepareHandoff(input: {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
chain: string[];
|
||||||
|
currentSkill: string;
|
||||||
|
depth: number;
|
||||||
|
handoff: HandoffRequest;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
}):
|
||||||
|
| {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
chain: string[];
|
||||||
|
depth: number;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
request: AgentRequest;
|
||||||
|
skill: Skill;
|
||||||
|
}
|
||||||
|
| null {
|
||||||
|
const target = this.resolveHandoffTarget(input);
|
||||||
|
const nextSkill = selectSkillForRequest(this.skills, {
|
||||||
|
...input.request,
|
||||||
|
skill: target.targetAgent,
|
||||||
|
trigger: "handoff",
|
||||||
|
});
|
||||||
|
if (!nextSkill) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
input.budget.recordHandoff(nextSkill.name);
|
||||||
|
metrics.handoffTotal.inc({
|
||||||
|
from: input.currentSkill,
|
||||||
|
reason: target.reason,
|
||||||
|
to: nextSkill.name,
|
||||||
|
});
|
||||||
|
|
||||||
|
const nextChain = [...input.chain, nextSkill.name];
|
||||||
|
const packet = buildHandoffPacket({
|
||||||
|
handoff: {
|
||||||
|
...input.handoff,
|
||||||
|
reason: target.reason,
|
||||||
|
targetAgent: nextSkill.name,
|
||||||
|
},
|
||||||
|
history: input.history,
|
||||||
|
request: input.request,
|
||||||
|
response: input.response,
|
||||||
|
sourceChain: nextChain,
|
||||||
|
state: {
|
||||||
|
source: input.currentSkill,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
budget: input.budget,
|
||||||
|
chain: nextChain,
|
||||||
|
depth: input.depth + 1,
|
||||||
|
history: input.history,
|
||||||
|
request: {
|
||||||
|
...input.request,
|
||||||
|
context: {
|
||||||
|
...(input.request.context ?? {}),
|
||||||
|
handoffPacket: packet,
|
||||||
|
},
|
||||||
|
skill: nextSkill.name,
|
||||||
|
trigger: "handoff",
|
||||||
|
},
|
||||||
|
skill: nextSkill,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveHandoffTarget(input: {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
currentSkill: string;
|
||||||
|
handoff: HandoffRequest;
|
||||||
|
}): { reason: HandoffReason; targetAgent: string } {
|
||||||
|
const requestedTarget = input.handoff.targetAgent;
|
||||||
|
const allowed = this.transitionMatrix?.isAllowed(
|
||||||
|
input.currentSkill,
|
||||||
|
requestedTarget,
|
||||||
|
) ?? true;
|
||||||
|
if (!allowed) {
|
||||||
|
return {
|
||||||
|
reason: "policy-risk",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
||||||
|
if (verdict === "ok") {
|
||||||
|
return {
|
||||||
|
reason: input.handoff.reason,
|
||||||
|
targetAgent: requestedTarget,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (verdict === "same-agent-loop") {
|
||||||
|
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
reason: budgetVerdictToReason(verdict),
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private makeSupervisorFallback(
|
||||||
|
request: AgentRequest,
|
||||||
|
reason: HandoffReason,
|
||||||
|
): AgentResponse {
|
||||||
|
return {
|
||||||
|
confidence: 0.4,
|
||||||
|
metadata: {
|
||||||
|
supervisorFallbackReason: reason,
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||||
|
if (verdict === "same-agent-loop") {
|
||||||
|
return "same-agent-revisit";
|
||||||
|
}
|
||||||
|
if (verdict === "max-hops") {
|
||||||
|
return "max-hop";
|
||||||
|
}
|
||||||
|
return "policy-risk";
|
||||||
|
}
|
||||||
|
|
||||||
|
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
||||||
|
const value = response.metadata?.["handoffRequest"];
|
||||||
|
if (!value || typeof value !== "object") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const handoff = value as Partial<HandoffRequest>;
|
||||||
|
if (
|
||||||
|
typeof handoff.targetAgent !== "string" ||
|
||||||
|
typeof handoff.reason !== "string" ||
|
||||||
|
typeof handoff.taskSpec !== "string" ||
|
||||||
|
typeof handoff.confidence !== "number" ||
|
||||||
|
typeof handoff.expectedOutput !== "string"
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
confidence: handoff.confidence,
|
||||||
|
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
||||||
|
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||||
|
: [],
|
||||||
|
expectedOutput: handoff.expectedOutput,
|
||||||
|
reason: handoff.reason,
|
||||||
|
risk:
|
||||||
|
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
||||||
|
? handoff.risk
|
||||||
|
: "medium",
|
||||||
|
targetAgent: handoff.targetAgent,
|
||||||
|
taskSpec: handoff.taskSpec,
|
||||||
|
...(handoff.userVisibleMessage
|
||||||
|
? { userVisibleMessage: handoff.userVisibleMessage }
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { BudgetTracker } from "./budget.js";
|
||||||
|
|
||||||
|
describe("BudgetTracker", () => {
|
||||||
|
it("allows handoff while within limits", () => {
|
||||||
|
const budget = new BudgetTracker({
|
||||||
|
enabled: true,
|
||||||
|
maxCostUsd: 1,
|
||||||
|
maxHandoffs: 2,
|
||||||
|
maxSameAgentVisits: 2,
|
||||||
|
maxToolCalls: 2,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(budget.assertBeforeHandoff("moderation")).toBe("ok");
|
||||||
|
budget.recordHandoff("moderation");
|
||||||
|
expect(budget.snapshot()).toMatchObject({
|
||||||
|
handoffs: 1,
|
||||||
|
visits: { moderation: 1 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects loops and exhausted handoff budgets", () => {
|
||||||
|
const budget = new BudgetTracker({
|
||||||
|
enabled: true,
|
||||||
|
maxCostUsd: 1,
|
||||||
|
maxHandoffs: 1,
|
||||||
|
maxSameAgentVisits: 1,
|
||||||
|
maxToolCalls: 5,
|
||||||
|
});
|
||||||
|
|
||||||
|
budget.recordVisit("moderation");
|
||||||
|
expect(budget.assertBeforeHandoff("moderation")).toBe("same-agent-loop");
|
||||||
|
|
||||||
|
budget.recordHandoff("onboarding");
|
||||||
|
expect(budget.assertBeforeHandoff("supervisor")).toBe("max-hops");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects exhausted tool and cost budgets", () => {
|
||||||
|
const budget = new BudgetTracker({
|
||||||
|
enabled: true,
|
||||||
|
maxCostUsd: 0.5,
|
||||||
|
maxHandoffs: 5,
|
||||||
|
maxSameAgentVisits: 5,
|
||||||
|
maxToolCalls: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
budget.recordToolCall();
|
||||||
|
expect(budget.assertBeforeHandoff("supervisor")).toBe("tool-budget");
|
||||||
|
|
||||||
|
const costBudget = new BudgetTracker({
|
||||||
|
enabled: true,
|
||||||
|
maxCostUsd: 0.5,
|
||||||
|
maxHandoffs: 5,
|
||||||
|
maxSameAgentVisits: 5,
|
||||||
|
maxToolCalls: 5,
|
||||||
|
});
|
||||||
|
costBudget.recordToolCall(0.5);
|
||||||
|
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("can be disabled", () => {
|
||||||
|
const budget = new BudgetTracker({
|
||||||
|
enabled: false,
|
||||||
|
maxCostUsd: 0,
|
||||||
|
maxHandoffs: 0,
|
||||||
|
maxSameAgentVisits: 0,
|
||||||
|
maxToolCalls: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
budget.recordVisit("supervisor");
|
||||||
|
budget.recordHandoff("moderation");
|
||||||
|
budget.recordToolCall(1);
|
||||||
|
|
||||||
|
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import {
|
||||||
|
getAgentBudgetEnabled,
|
||||||
|
getAgentBudgetMaxCostUsd,
|
||||||
|
getAgentBudgetMaxHandoffs,
|
||||||
|
getAgentBudgetMaxSameAgentVisits,
|
||||||
|
getAgentBudgetMaxToolCalls,
|
||||||
|
} from "../config.js";
|
||||||
|
|
||||||
|
export type BudgetVerdict =
|
||||||
|
| "ok"
|
||||||
|
| "max-hops"
|
||||||
|
| "same-agent-loop"
|
||||||
|
| "tool-budget"
|
||||||
|
| "cost-budget";
|
||||||
|
|
||||||
|
export interface BudgetSnapshot {
|
||||||
|
estimatedCostUsd: number;
|
||||||
|
handoffs: number;
|
||||||
|
maxCostUsd: number;
|
||||||
|
maxHandoffs: number;
|
||||||
|
maxSameAgentVisits: number;
|
||||||
|
maxToolCalls: number;
|
||||||
|
toolCalls: number;
|
||||||
|
visits: Record<string, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class BudgetTracker {
|
||||||
|
private estimatedCostUsd = 0;
|
||||||
|
private handoffs = 0;
|
||||||
|
private toolCalls = 0;
|
||||||
|
private readonly visits = new Map<string, number>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly limits = {
|
||||||
|
enabled: getAgentBudgetEnabled(),
|
||||||
|
maxCostUsd: getAgentBudgetMaxCostUsd(),
|
||||||
|
maxHandoffs: getAgentBudgetMaxHandoffs(),
|
||||||
|
maxSameAgentVisits: getAgentBudgetMaxSameAgentVisits(),
|
||||||
|
maxToolCalls: getAgentBudgetMaxToolCalls(),
|
||||||
|
},
|
||||||
|
) {}
|
||||||
|
|
||||||
|
assertBeforeHandoff(target: string): BudgetVerdict {
|
||||||
|
if (!this.limits.enabled) {
|
||||||
|
return "ok";
|
||||||
|
}
|
||||||
|
if (this.handoffs >= this.limits.maxHandoffs) {
|
||||||
|
return "max-hops";
|
||||||
|
}
|
||||||
|
if ((this.visits.get(target) ?? 0) >= this.limits.maxSameAgentVisits) {
|
||||||
|
return "same-agent-loop";
|
||||||
|
}
|
||||||
|
if (this.toolCalls >= this.limits.maxToolCalls) {
|
||||||
|
return "tool-budget";
|
||||||
|
}
|
||||||
|
if (this.estimatedCostUsd >= this.limits.maxCostUsd) {
|
||||||
|
return "cost-budget";
|
||||||
|
}
|
||||||
|
return "ok";
|
||||||
|
}
|
||||||
|
|
||||||
|
recordHandoff(target: string): void {
|
||||||
|
this.handoffs += 1;
|
||||||
|
this.recordVisit(target);
|
||||||
|
}
|
||||||
|
|
||||||
|
recordToolCall(estimatedCostUsd = 0): void {
|
||||||
|
this.toolCalls += 1;
|
||||||
|
this.estimatedCostUsd += estimatedCostUsd;
|
||||||
|
}
|
||||||
|
|
||||||
|
recordVisit(target: string): void {
|
||||||
|
this.visits.set(target, (this.visits.get(target) ?? 0) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot(): BudgetSnapshot {
|
||||||
|
return {
|
||||||
|
estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)),
|
||||||
|
handoffs: this.handoffs,
|
||||||
|
maxCostUsd: this.limits.maxCostUsd,
|
||||||
|
maxHandoffs: this.limits.maxHandoffs,
|
||||||
|
maxSameAgentVisits: this.limits.maxSameAgentVisits,
|
||||||
|
maxToolCalls: this.limits.maxToolCalls,
|
||||||
|
toolCalls: this.toolCalls,
|
||||||
|
visits: Object.fromEntries(this.visits.entries()),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { buildHandoffPacket, readHandoffPacket } from "./handoff-packet.js";
|
||||||
|
import type { AgentRequest, AgentResponse, ConversationMessage } from "./types.js";
|
||||||
|
|
||||||
|
const request: AgentRequest = {
|
||||||
|
chatroomId: "room-1",
|
||||||
|
nickname: "tester",
|
||||||
|
requestId: "request-1",
|
||||||
|
skill: "onboarding",
|
||||||
|
trigger: "user-message",
|
||||||
|
userId: "user-1",
|
||||||
|
userMessage: "private original text",
|
||||||
|
};
|
||||||
|
|
||||||
|
const response: AgentResponse = {
|
||||||
|
confidence: 0.7,
|
||||||
|
requestId: "request-1",
|
||||||
|
source: "template",
|
||||||
|
text: "needs handoff",
|
||||||
|
};
|
||||||
|
|
||||||
|
const history: ConversationMessage[] = [
|
||||||
|
{
|
||||||
|
content: "이전 메시지",
|
||||||
|
role: "user",
|
||||||
|
timestamp: "2026-05-18T00:00:00.000Z",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
describe("handoff packet", () => {
|
||||||
|
it("builds a bounded packet without raw user message", () => {
|
||||||
|
const packet = buildHandoffPacket({
|
||||||
|
handoff: {
|
||||||
|
confidence: 0.7,
|
||||||
|
evidenceRefs: ["tool-1"],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "out-of-scope",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "moderation",
|
||||||
|
taskSpec: "Call 010-1234-5678 after 2026-05-18",
|
||||||
|
},
|
||||||
|
history,
|
||||||
|
request,
|
||||||
|
response,
|
||||||
|
sourceChain: ["onboarding", "moderation"],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(packet.userGoal).toBe("Call <phone> after <date>");
|
||||||
|
expect(packet.userGoal).not.toContain(request.userMessage);
|
||||||
|
expect(packet.sourceChain).toEqual(["onboarding", "moderation"]);
|
||||||
|
expect(packet.evidence).toEqual([
|
||||||
|
{
|
||||||
|
id: "tool-1",
|
||||||
|
kind: "tool-call",
|
||||||
|
summary: "source agent evidence reference",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads packet from request context", () => {
|
||||||
|
const packet = buildHandoffPacket({
|
||||||
|
handoff: {
|
||||||
|
confidence: 0.7,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "missing-tool",
|
||||||
|
risk: "low",
|
||||||
|
targetAgent: "moderation",
|
||||||
|
taskSpec: "continue",
|
||||||
|
},
|
||||||
|
request,
|
||||||
|
response,
|
||||||
|
sourceChain: ["onboarding", "moderation"],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(readHandoffPacket({ ...request, context: { handoffPacket: packet } })).toBe(packet);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import type {
|
||||||
|
AgentRequest,
|
||||||
|
AgentResponse,
|
||||||
|
ConversationMessage,
|
||||||
|
HandoffReason,
|
||||||
|
HandoffRequest,
|
||||||
|
} from "./types.js";
|
||||||
|
|
||||||
|
export interface HandoffEvidence {
|
||||||
|
id: string;
|
||||||
|
kind: "message" | "tool-call" | "response";
|
||||||
|
summary: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HandoffPacket {
|
||||||
|
constraints: string[];
|
||||||
|
currentState: Record<string, unknown>;
|
||||||
|
evidence: HandoffEvidence[];
|
||||||
|
expectedOutputSchema: string;
|
||||||
|
handoffReason: HandoffReason;
|
||||||
|
sourceChain: string[];
|
||||||
|
summary: string;
|
||||||
|
userGoal: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function asRecord(value: unknown): Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null
|
||||||
|
? (value as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
}
|
||||||
|
|
||||||
|
function fallbackSummary(history: readonly ConversationMessage[]): string {
|
||||||
|
return history
|
||||||
|
.slice(-3)
|
||||||
|
.map((message) => `${message.role}: ${message.content.slice(0, 120)}`)
|
||||||
|
.join(" | ");
|
||||||
|
}
|
||||||
|
|
||||||
|
function sanitizeGoal(value: string | undefined): string {
|
||||||
|
if (!value?.trim()) {
|
||||||
|
return "이전 agent가 요청한 작업을 이어서 처리";
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
.replace(/\b\d{2,4}[-./]\d{1,2}[-./]\d{1,2}\b/gu, "<date>")
|
||||||
|
.replace(/\b\d{2,3}[- ]?\d{3,4}[- ]?\d{4}\b/gu, "<phone>")
|
||||||
|
.slice(0, 240);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildHandoffPacket(input: {
|
||||||
|
handoff: HandoffRequest;
|
||||||
|
history?: readonly ConversationMessage[] | undefined;
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
sourceChain: string[];
|
||||||
|
state?: Record<string, unknown> | undefined;
|
||||||
|
summary?: string | undefined;
|
||||||
|
}): HandoffPacket {
|
||||||
|
return {
|
||||||
|
constraints: [
|
||||||
|
"원문 사용자 메시지와 닉네임 등 PII를 새로 노출하지 않는다.",
|
||||||
|
"handoffRequest.expectedOutput 범위 안에서만 답한다.",
|
||||||
|
],
|
||||||
|
currentState: input.state ?? {},
|
||||||
|
evidence: input.handoff.evidenceRefs.map((id) => ({
|
||||||
|
id,
|
||||||
|
kind: "tool-call",
|
||||||
|
summary: "source agent evidence reference",
|
||||||
|
})),
|
||||||
|
expectedOutputSchema: input.handoff.expectedOutput,
|
||||||
|
handoffReason: input.handoff.reason,
|
||||||
|
sourceChain: input.sourceChain,
|
||||||
|
summary: input.summary?.trim() || fallbackSummary(input.history ?? []),
|
||||||
|
userGoal: sanitizeGoal(input.handoff.taskSpec),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readHandoffPacket(request: AgentRequest): HandoffPacket | undefined {
|
||||||
|
const packet = asRecord(request.context)["handoffPacket"];
|
||||||
|
if (!packet || typeof packet !== "object") {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return packet as HandoffPacket;
|
||||||
|
}
|
||||||
@@ -9,6 +9,27 @@ export interface AgentRequest {
|
|||||||
context?: Record<string, unknown> | undefined;
|
context?: Record<string, unknown> | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type HandoffReason =
|
||||||
|
| "out-of-scope"
|
||||||
|
| "missing-tool"
|
||||||
|
| "low-confidence"
|
||||||
|
| "validation-failed"
|
||||||
|
| "same-agent-revisit"
|
||||||
|
| "max-hop"
|
||||||
|
| "high-risk-action"
|
||||||
|
| "policy-risk";
|
||||||
|
|
||||||
|
export interface HandoffRequest {
|
||||||
|
targetAgent: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
taskSpec: string;
|
||||||
|
evidenceRefs: string[];
|
||||||
|
confidence: number;
|
||||||
|
risk: "low" | "medium" | "high";
|
||||||
|
expectedOutput: string;
|
||||||
|
userVisibleMessage?: string | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
export interface AgentResponse {
|
export interface AgentResponse {
|
||||||
requestId: string;
|
requestId: string;
|
||||||
text: string;
|
text: string;
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ export interface AlertPayload {
|
|||||||
| "eval-drift"
|
| "eval-drift"
|
||||||
| "eval-score-drop"
|
| "eval-score-drop"
|
||||||
| "latency"
|
| "latency"
|
||||||
| "llm-error-rate";
|
| "llm-error-rate"
|
||||||
|
| "supervisor-handoff";
|
||||||
metric: string;
|
metric: string;
|
||||||
severity: "info" | "warn" | "critical";
|
severity: "info" | "warn" | "critical";
|
||||||
suggestedAction?: string | undefined;
|
suggestedAction?: string | undefined;
|
||||||
|
|||||||
@@ -98,6 +98,34 @@ export function getAgentRequestTimeoutMs(): number {
|
|||||||
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
|
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getAgentBudgetEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_BUDGET_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentBudgetMaxHandoffs(): number {
|
||||||
|
return getPositiveNumber("AGENT_BUDGET_MAX_HANDOFFS", 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentBudgetMaxSameAgentVisits(): number {
|
||||||
|
return getPositiveNumber("AGENT_BUDGET_MAX_SAME_AGENT_VISITS", 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentBudgetMaxToolCalls(): number {
|
||||||
|
return getPositiveNumber("AGENT_BUDGET_MAX_TOOL_CALLS", 20);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentBudgetMaxCostUsd(): number {
|
||||||
|
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentRuntimeVerifierEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getMetricsPort(): number {
|
||||||
|
return getPositiveNumber("METRICS_PORT", 9464);
|
||||||
|
}
|
||||||
|
|
||||||
export function getDiscordVerifyTimeoutMs(): number {
|
export function getDiscordVerifyTimeoutMs(): number {
|
||||||
return getPositiveNumber("DISCORD_VERIFY_TIMEOUT_MS", 2_500);
|
return getPositiveNumber("DISCORD_VERIFY_TIMEOUT_MS", 2_500);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||||
|
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||||
|
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||||
|
|
||||||
|
export type RuntimeVerdict = "pass" | "retry" | "escalate";
|
||||||
|
|
||||||
|
export interface RuntimeVerifierInput {
|
||||||
|
derived?: unknown;
|
||||||
|
packet?: unknown;
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
skill: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function verifyOnboardingRuntime(
|
||||||
|
input: RuntimeVerifierInput,
|
||||||
|
): Promise<RuntimeVerdict> {
|
||||||
|
const derived = input.derived as DerivedState | undefined;
|
||||||
|
if (!derived) {
|
||||||
|
return Promise.resolve("pass");
|
||||||
|
}
|
||||||
|
const validation = validateOnboardingResponse(
|
||||||
|
input.response.text.trim() ? input.response.text : "[NO_RESPONSE]",
|
||||||
|
derived,
|
||||||
|
{
|
||||||
|
allowShortResponse: derived.allowShortResponse,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return Promise.resolve(validation.status === "reject" ? "retry" : "pass");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function verifyModerationRuntime(
|
||||||
|
input: RuntimeVerifierInput,
|
||||||
|
): Promise<RuntimeVerdict> {
|
||||||
|
const text = input.response.text;
|
||||||
|
if (/https?:\/\//iu.test(text) || /\b\d{2,3}[- ]?\d{3,4}[- ]?\d{4}\b/u.test(text)) {
|
||||||
|
return Promise.resolve("escalate");
|
||||||
|
}
|
||||||
|
return Promise.resolve("pass");
|
||||||
|
}
|
||||||
@@ -6,6 +6,12 @@ import {
|
|||||||
type ValidationResult,
|
type ValidationResult,
|
||||||
} from "@skills/onboarding/runtime/validator";
|
} from "@skills/onboarding/runtime/validator";
|
||||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||||
|
import {
|
||||||
|
verifyModerationRuntime,
|
||||||
|
verifyOnboardingRuntime,
|
||||||
|
type RuntimeVerifierInput,
|
||||||
|
type RuntimeVerdict,
|
||||||
|
} from "./runtime-verifier.js";
|
||||||
|
|
||||||
export type SkillValidationResult = ValidationResult;
|
export type SkillValidationResult = ValidationResult;
|
||||||
|
|
||||||
@@ -14,6 +20,7 @@ export interface SkillJudgeConfig {
|
|||||||
promptTemplate?: string | undefined;
|
promptTemplate?: string | undefined;
|
||||||
toolRegistryFactory: () => ToolRegistry;
|
toolRegistryFactory: () => ToolRegistry;
|
||||||
tournamentInstruction: string;
|
tournamentInstruction: string;
|
||||||
|
runtimeVerifier?: ((input: RuntimeVerifierInput) => Promise<RuntimeVerdict>) | undefined;
|
||||||
validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined;
|
validator?: ((text: string, derived: unknown) => SkillValidationResult) | undefined;
|
||||||
weights?: Record<string, number> | undefined;
|
weights?: Record<string, number> | undefined;
|
||||||
}
|
}
|
||||||
@@ -24,12 +31,14 @@ export const SKILL_JUDGE_CONFIGS: Record<string, SkillJudgeConfig> = {
|
|||||||
toolRegistryFactory: createModerationToolRegistry,
|
toolRegistryFactory: createModerationToolRegistry,
|
||||||
tournamentInstruction:
|
tournamentInstruction:
|
||||||
"커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.",
|
"커뮤니티 규칙/운영 문의 응답으로서 적절성, 정확성, 도움됨을 비교하세요.",
|
||||||
|
runtimeVerifier: verifyModerationRuntime,
|
||||||
},
|
},
|
||||||
onboarding: {
|
onboarding: {
|
||||||
axes: ["friendliness", "accuracy", "concision", "scope"],
|
axes: ["friendliness", "accuracy", "concision", "scope"],
|
||||||
toolRegistryFactory: createOnboardingToolRegistry,
|
toolRegistryFactory: createOnboardingToolRegistry,
|
||||||
tournamentInstruction:
|
tournamentInstruction:
|
||||||
"온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.",
|
"온보딩 응답으로서 친절함, 정확성, 간결함, 질문 범위 준수를 비교하세요.",
|
||||||
|
runtimeVerifier: verifyOnboardingRuntime,
|
||||||
validator: (text, derived) =>
|
validator: (text, derived) =>
|
||||||
validateOnboardingResponse(text, derived as DerivedState, {
|
validateOnboardingResponse(text, derived as DerivedState, {
|
||||||
allowShortResponse: (derived as DerivedState).allowShortResponse,
|
allowShortResponse: (derived as DerivedState).allowShortResponse,
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
const MODEL_RATES: Record<string, { inputPer1M: number; outputPer1M: number }> = {
|
||||||
|
"gpt-5.3-codex-spark": { inputPer1M: 0.15, outputPer1M: 0.6 },
|
||||||
|
"google/gemma-4-26b-a4b-it:free": { inputPer1M: 0, outputPer1M: 0 },
|
||||||
|
"google/gemma-4-31b-it:free": { inputPer1M: 0, outputPer1M: 0 },
|
||||||
|
};
|
||||||
|
|
||||||
|
export function estimateChatCost(
|
||||||
|
model: string,
|
||||||
|
tokensIn = 0,
|
||||||
|
tokensOut = 0,
|
||||||
|
): number {
|
||||||
|
const rate = MODEL_RATES[model] ?? { inputPer1M: 0.1, outputPer1M: 0.3 };
|
||||||
|
return Number(
|
||||||
|
((tokensIn / 1_000_000) * rate.inputPer1M +
|
||||||
|
(tokensOut / 1_000_000) * rate.outputPer1M).toFixed(6),
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -17,6 +17,8 @@ import {
|
|||||||
import { LlmCircuitBreaker } from "./circuit.js";
|
import { LlmCircuitBreaker } from "./circuit.js";
|
||||||
import { getProviderAdapter } from "./providers/index.js";
|
import { getProviderAdapter } from "./providers/index.js";
|
||||||
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
|
import { selectMaxTokensForTier, selectModelsForProvider, selectProviderOrder } from "./routing.js";
|
||||||
|
import { estimateChatCost } from "./cost.js";
|
||||||
|
import { metrics } from "@obs/metrics";
|
||||||
|
|
||||||
const DEFAULT_MODEL = "Qwen3.5-4B";
|
const DEFAULT_MODEL = "Qwen3.5-4B";
|
||||||
|
|
||||||
@@ -211,6 +213,13 @@ export class LlmClient {
|
|||||||
|
|
||||||
const payload = (await response.json()) as ChatCompletionResponse;
|
const payload = (await response.json()) as ChatCompletionResponse;
|
||||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||||
|
metrics.tokenCostUsd.observe(
|
||||||
|
estimateChatCost(
|
||||||
|
request.model,
|
||||||
|
payload.usage?.prompt_tokens,
|
||||||
|
payload.usage?.completion_tokens,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
span.setAttributes({
|
span.setAttributes({
|
||||||
"gen_ai.response.finish_reasons": payload.choices
|
"gen_ai.response.finish_reasons": payload.choices
|
||||||
|
|||||||
+11
-2
@@ -23,12 +23,17 @@ const { Agent } = await import("./agent/index.js");
|
|||||||
const { ConversationStore } = await import("./conversation/index.js");
|
const { ConversationStore } = await import("./conversation/index.js");
|
||||||
const { createOnboardingSkill } = await import("./skills/onboarding/index.js");
|
const { createOnboardingSkill } = await import("./skills/onboarding/index.js");
|
||||||
const { createModerationSkill } = await import("./skills/moderation/index.js");
|
const { createModerationSkill } = await import("./skills/moderation/index.js");
|
||||||
|
const { createSupervisorSkill } = await import("./skills/supervisor/index.js");
|
||||||
const { SkillRegistry } = await import("./skills/skill-loader.js");
|
const { SkillRegistry } = await import("./skills/skill-loader.js");
|
||||||
|
const { TransitionMatrix } = await import("./skills/transition-matrix.js");
|
||||||
const { ShadowEvalBus } = await import("./eval/shadow-bus.js");
|
const { ShadowEvalBus } = await import("./eval/shadow-bus.js");
|
||||||
const { LlmClient } = await import("./llm/index.js");
|
const { LlmClient } = await import("./llm/index.js");
|
||||||
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
|
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
|
||||||
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
|
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
|
||||||
const { RoomContextService } = await import("./room-context/index.js");
|
const { RoomContextService } = await import("./room-context/index.js");
|
||||||
|
const { startMetricsServer, stopMetricsServer } = await import(
|
||||||
|
"./observability/metrics-server.js"
|
||||||
|
);
|
||||||
const { resolve } = await import("node:path");
|
const { resolve } = await import("node:path");
|
||||||
|
|
||||||
function firstNonEmpty(
|
function firstNonEmpty(
|
||||||
@@ -219,14 +224,16 @@ async function bootstrap(): Promise<void> {
|
|||||||
|
|
||||||
// Agent setup
|
// Agent setup
|
||||||
const conversationStore = new ConversationStore(garnetClient);
|
const conversationStore = new ConversationStore(garnetClient);
|
||||||
const agent = new Agent(natsClient, conversationStore);
|
const skillRegistry = SkillRegistry.load(resolve(process.cwd(), "src", "skills"));
|
||||||
|
const transitionMatrix = TransitionMatrix.fromRegistry(skillRegistry);
|
||||||
|
const agent = new Agent(natsClient, conversationStore, transitionMatrix);
|
||||||
|
|
||||||
// Register skills
|
// Register skills
|
||||||
const shadowEvalBus = new ShadowEvalBus(garnetClient);
|
const shadowEvalBus = new ShadowEvalBus(garnetClient);
|
||||||
const skillRegistry = SkillRegistry.load(resolve(process.cwd(), "src", "skills"));
|
|
||||||
const factories: Record<string, SkillFactory> = {
|
const factories: Record<string, SkillFactory> = {
|
||||||
moderation: createModerationSkill,
|
moderation: createModerationSkill,
|
||||||
onboarding: createOnboardingSkill,
|
onboarding: createOnboardingSkill,
|
||||||
|
supervisor: createSupervisorSkill,
|
||||||
};
|
};
|
||||||
const seenSkills = new Set<string>();
|
const seenSkills = new Set<string>();
|
||||||
for (const manifest of skillRegistry.list()) {
|
for (const manifest of skillRegistry.list()) {
|
||||||
@@ -260,12 +267,14 @@ async function bootstrap(): Promise<void> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
installShutdownHooks(async () => {
|
installShutdownHooks(async () => {
|
||||||
|
await stopMetricsServer();
|
||||||
await natsClient.close();
|
await natsClient.close();
|
||||||
await garnetClient.close();
|
await garnetClient.close();
|
||||||
await closeMongoDB();
|
await closeMongoDB();
|
||||||
await shutdownTracing();
|
await shutdownTracing();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
startMetricsServer();
|
||||||
await agent.start();
|
await agent.start();
|
||||||
|
|
||||||
logger.info(`✓ ${serviceName} bootstrapped`);
|
logger.info(`✓ ${serviceName} bootstrapped`);
|
||||||
|
|||||||
@@ -3,10 +3,14 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
|||||||
import type { Logger } from 'pino';
|
import type { Logger } from 'pino';
|
||||||
|
|
||||||
import { getLogger } from './logger.js';
|
import { getLogger } from './logger.js';
|
||||||
|
import type { BudgetSnapshot } from '@agent/budget';
|
||||||
|
|
||||||
export interface RequestContext {
|
export interface RequestContext {
|
||||||
chatroomId: string;
|
chatroomId: string;
|
||||||
deadlineAt?: string;
|
deadlineAt?: string;
|
||||||
|
budget?: BudgetSnapshot | undefined;
|
||||||
|
handoffChain?: string[] | undefined;
|
||||||
|
handoffDepth?: number | undefined;
|
||||||
logger: Logger;
|
logger: Logger;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
skill: string;
|
skill: string;
|
||||||
@@ -27,6 +31,7 @@ export function runWithContext<T>(
|
|||||||
chatroomId: context.chatroomId,
|
chatroomId: context.chatroomId,
|
||||||
requestId: context.requestId,
|
requestId: context.requestId,
|
||||||
skill: context.skill,
|
skill: context.skill,
|
||||||
|
handoffDepth: context.handoffDepth,
|
||||||
trigger: context.trigger,
|
trigger: context.trigger,
|
||||||
userId: context.userId,
|
userId: context.userId,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { getLogLevel, getNodeEnv, getServiceName } from '../config.js';
|
|||||||
|
|
||||||
type RequestBindings = {
|
type RequestBindings = {
|
||||||
chatroomId?: string;
|
chatroomId?: string;
|
||||||
|
handoffDepth?: number | undefined;
|
||||||
requestId?: string;
|
requestId?: string;
|
||||||
skill?: string;
|
skill?: string;
|
||||||
trigger?: string;
|
trigger?: string;
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import { createServer, type Server } from "node:http";
|
||||||
|
|
||||||
|
import { getMetricsPort } from "../config.js";
|
||||||
|
import { getLogger } from "./logger.js";
|
||||||
|
import { renderMetrics } from "./metrics.js";
|
||||||
|
|
||||||
|
let server: Server | null = null;
|
||||||
|
|
||||||
|
export function startMetricsServer(port = getMetricsPort()): Server {
|
||||||
|
if (server) {
|
||||||
|
return server;
|
||||||
|
}
|
||||||
|
|
||||||
|
server = createServer((request, response) => {
|
||||||
|
if (request.url !== "/metrics") {
|
||||||
|
response.writeHead(404);
|
||||||
|
response.end("not found");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
response.writeHead(200, {
|
||||||
|
"Content-Type": "text/plain; version=0.0.4",
|
||||||
|
});
|
||||||
|
response.end(renderMetrics());
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(port, () => {
|
||||||
|
getLogger().info({ port }, "✓ Metrics server listening");
|
||||||
|
});
|
||||||
|
|
||||||
|
return server;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function stopMetricsServer(): Promise<void> {
|
||||||
|
if (!server) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const current = server;
|
||||||
|
server = null;
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
current.close((error) => {
|
||||||
|
if (error) {
|
||||||
|
reject(error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { metrics, renderMetrics } from "./metrics.js";
|
||||||
|
|
||||||
|
describe("metrics", () => {
|
||||||
|
it("renders the agent metric families", () => {
|
||||||
|
const body = renderMetrics();
|
||||||
|
|
||||||
|
for (const name of [
|
||||||
|
"agent_routing_accuracy_total",
|
||||||
|
"agent_handoff_total",
|
||||||
|
"agent_handoff_loop_total",
|
||||||
|
"agent_handoff_count",
|
||||||
|
"agent_unnecessary_escalation_total",
|
||||||
|
"agent_missed_escalation_total",
|
||||||
|
"agent_token_cost_usd",
|
||||||
|
"agent_time_to_resolution_ms",
|
||||||
|
"agent_validator_fail_total",
|
||||||
|
"agent_human_override_total",
|
||||||
|
]) {
|
||||||
|
expect(body).toContain(`# HELP ${name}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders counter labels and histogram observations", () => {
|
||||||
|
metrics.handoffTotal.inc({
|
||||||
|
from: "onboarding",
|
||||||
|
reason: "out-of-scope",
|
||||||
|
to: "moderation",
|
||||||
|
});
|
||||||
|
metrics.handoffCount.observe(2);
|
||||||
|
|
||||||
|
const body = renderMetrics();
|
||||||
|
|
||||||
|
expect(body).toContain(
|
||||||
|
'agent_handoff_total{from="onboarding",reason="out-of-scope",to="moderation"} 1',
|
||||||
|
);
|
||||||
|
expect(body).toContain('agent_handoff_count_bucket{le="2"} 1');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
type Labels = Record<string, string | number | undefined>;
|
||||||
|
|
||||||
|
interface MetricSample {
|
||||||
|
labels: Labels;
|
||||||
|
value: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function labelKey(labels: Labels): string {
|
||||||
|
return JSON.stringify(
|
||||||
|
Object.fromEntries(
|
||||||
|
Object.entries(labels)
|
||||||
|
.filter(([, value]) => value !== undefined)
|
||||||
|
.sort(([left], [right]) => left.localeCompare(right)),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatLabels(labels: Labels): string {
|
||||||
|
const entries = Object.entries(labels).filter(([, value]) => value !== undefined);
|
||||||
|
if (entries.length === 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return `{${entries
|
||||||
|
.map(([key, value]) => `${key}="${String(value).replace(/"/gu, '\\"')}"`)
|
||||||
|
.join(",")}}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
class CounterMetric {
|
||||||
|
private readonly samples = new Map<string, MetricSample>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
readonly name: string,
|
||||||
|
readonly help: string,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
inc(labels: Labels = {}, value = 1): void {
|
||||||
|
const key = labelKey(labels);
|
||||||
|
const sample = this.samples.get(key) ?? { labels, value: 0 };
|
||||||
|
sample.value += value;
|
||||||
|
this.samples.set(key, sample);
|
||||||
|
}
|
||||||
|
|
||||||
|
render(): string[] {
|
||||||
|
return [
|
||||||
|
`# HELP ${this.name} ${this.help}`,
|
||||||
|
`# TYPE ${this.name} counter`,
|
||||||
|
...[...this.samples.values()].map(
|
||||||
|
(sample) => `${this.name}${formatLabels(sample.labels)} ${sample.value}`,
|
||||||
|
),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class HistogramMetric {
|
||||||
|
private readonly values: number[] = [];
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
readonly name: string,
|
||||||
|
readonly help: string,
|
||||||
|
private readonly buckets: number[],
|
||||||
|
) {}
|
||||||
|
|
||||||
|
observe(value: number): void {
|
||||||
|
if (Number.isFinite(value)) {
|
||||||
|
this.values.push(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
render(): string[] {
|
||||||
|
const lines = [
|
||||||
|
`# HELP ${this.name} ${this.help}`,
|
||||||
|
`# TYPE ${this.name} histogram`,
|
||||||
|
];
|
||||||
|
for (const bucket of this.buckets) {
|
||||||
|
const count = this.values.filter((value) => value <= bucket).length;
|
||||||
|
lines.push(`${this.name}_bucket{le="${bucket}"} ${count}`);
|
||||||
|
}
|
||||||
|
lines.push(`${this.name}_bucket{le="+Inf"} ${this.values.length}`);
|
||||||
|
lines.push(
|
||||||
|
`${this.name}_sum ${this.values.reduce((sum, value) => sum + value, 0)}`,
|
||||||
|
);
|
||||||
|
lines.push(`${this.name}_count ${this.values.length}`);
|
||||||
|
return lines;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const metrics = {
|
||||||
|
handoffTotal: new CounterMetric("agent_handoff_total", "Agent handoffs"),
|
||||||
|
handoffLoopTotal: new CounterMetric(
|
||||||
|
"agent_handoff_loop_total",
|
||||||
|
"Agent handoff loops",
|
||||||
|
),
|
||||||
|
humanOverrideTotal: new CounterMetric(
|
||||||
|
"agent_human_override_total",
|
||||||
|
"Human override events",
|
||||||
|
),
|
||||||
|
missedEscalationTotal: new CounterMetric(
|
||||||
|
"agent_missed_escalation_total",
|
||||||
|
"Missed escalations",
|
||||||
|
),
|
||||||
|
routingAccuracyTotal: new CounterMetric(
|
||||||
|
"agent_routing_accuracy_total",
|
||||||
|
"Eval routing accuracy",
|
||||||
|
),
|
||||||
|
unnecessaryEscalationTotal: new CounterMetric(
|
||||||
|
"agent_unnecessary_escalation_total",
|
||||||
|
"Unnecessary escalations",
|
||||||
|
),
|
||||||
|
validatorFailTotal: new CounterMetric(
|
||||||
|
"agent_validator_fail_total",
|
||||||
|
"Validator failures",
|
||||||
|
),
|
||||||
|
handoffCount: new HistogramMetric(
|
||||||
|
"agent_handoff_count",
|
||||||
|
"Handoffs per request",
|
||||||
|
[0, 1, 2, 3, 4],
|
||||||
|
),
|
||||||
|
timeToResolutionMs: new HistogramMetric(
|
||||||
|
"agent_time_to_resolution_ms",
|
||||||
|
"Time to resolution",
|
||||||
|
[100, 500, 1_000, 2_500, 5_000, 10_000],
|
||||||
|
),
|
||||||
|
tokenCostUsd: new HistogramMetric(
|
||||||
|
"agent_token_cost_usd",
|
||||||
|
"Estimated LLM token cost USD",
|
||||||
|
[0.001, 0.01, 0.05, 0.1, 0.5, 1],
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
export function renderMetrics(): string {
|
||||||
|
return Object.values(metrics)
|
||||||
|
.flatMap((metric) => metric.render())
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { parseControlTokens } from "./control-token-parser.js";
|
||||||
|
|
||||||
|
describe("parseControlTokens", () => {
|
||||||
|
it("extracts handoff control tokens and strips them from text", () => {
|
||||||
|
const parsed = parseControlTokens(
|
||||||
|
"잠시 확인할게요. [HANDOFF:moderation:policy-risk]",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(parsed.stripped).toBe("잠시 확인할게요.");
|
||||||
|
expect(parsed.tokens).toEqual([
|
||||||
|
{
|
||||||
|
kind: "handoff",
|
||||||
|
raw: "[HANDOFF:moderation:policy-risk]",
|
||||||
|
reason: "policy-risk",
|
||||||
|
targetAgent: "moderation",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts no-response tokens", () => {
|
||||||
|
const parsed = parseControlTokens("[NO_RESPONSE]");
|
||||||
|
|
||||||
|
expect(parsed.stripped).toBe("");
|
||||||
|
expect(parsed.tokens).toEqual([
|
||||||
|
{
|
||||||
|
kind: "no-response",
|
||||||
|
raw: "[NO_RESPONSE]",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps invalid handoff reasons visible", () => {
|
||||||
|
const parsed = parseControlTokens("[HANDOFF:moderation:not-a-reason]");
|
||||||
|
|
||||||
|
expect(parsed.stripped).toBe("[HANDOFF:moderation:not-a-reason]");
|
||||||
|
expect(parsed.tokens).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accepts all handoff reason codes", () => {
|
||||||
|
const reasons = [
|
||||||
|
"out-of-scope",
|
||||||
|
"missing-tool",
|
||||||
|
"low-confidence",
|
||||||
|
"validation-failed",
|
||||||
|
"same-agent-revisit",
|
||||||
|
"max-hop",
|
||||||
|
"high-risk-action",
|
||||||
|
"policy-risk",
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const reason of reasons) {
|
||||||
|
const parsed = parseControlTokens(`[HANDOFF:supervisor:${reason}]`);
|
||||||
|
expect(parsed.tokens[0]).toMatchObject({
|
||||||
|
kind: "handoff",
|
||||||
|
reason,
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import type { HandoffReason } from "@agent/types";
|
||||||
|
|
||||||
|
const HANDOFF_REASONS = new Set<HandoffReason>([
|
||||||
|
"out-of-scope",
|
||||||
|
"missing-tool",
|
||||||
|
"low-confidence",
|
||||||
|
"validation-failed",
|
||||||
|
"same-agent-revisit",
|
||||||
|
"max-hop",
|
||||||
|
"high-risk-action",
|
||||||
|
"policy-risk",
|
||||||
|
]);
|
||||||
|
|
||||||
|
export type ControlToken =
|
||||||
|
| { kind: "no-response"; raw: string }
|
||||||
|
| {
|
||||||
|
kind: "handoff";
|
||||||
|
raw: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
targetAgent: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function parseControlTokens(text: string): {
|
||||||
|
stripped: string;
|
||||||
|
tokens: ControlToken[];
|
||||||
|
} {
|
||||||
|
const tokens: ControlToken[] = [];
|
||||||
|
let stripped = text;
|
||||||
|
|
||||||
|
stripped = stripped.replace(/\[NO_RESPONSE\]/gu, (raw) => {
|
||||||
|
tokens.push({ kind: "no-response", raw });
|
||||||
|
return "";
|
||||||
|
});
|
||||||
|
|
||||||
|
stripped = stripped.replace(
|
||||||
|
/\[HANDOFF:([a-z0-9_-]+):([a-z-]+)\]/giu,
|
||||||
|
(raw, target: string, reason: string) => {
|
||||||
|
if (!HANDOFF_REASONS.has(reason as HandoffReason)) {
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
tokens.push({
|
||||||
|
kind: "handoff",
|
||||||
|
raw,
|
||||||
|
reason: reason as HandoffReason,
|
||||||
|
targetAgent: target,
|
||||||
|
});
|
||||||
|
return "";
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
stripped: stripped.trim(),
|
||||||
|
tokens,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import type { HandoffReason } from "@agent/types";
|
||||||
|
import { decide } from "./trigger.js";
|
||||||
|
|
||||||
|
describe("escalation trigger", () => {
|
||||||
|
it("turns explicit handoff reasons into structured requests", () => {
|
||||||
|
const reasons: HandoffReason[] = [
|
||||||
|
"out-of-scope",
|
||||||
|
"missing-tool",
|
||||||
|
"low-confidence",
|
||||||
|
"validation-failed",
|
||||||
|
"same-agent-revisit",
|
||||||
|
"max-hop",
|
||||||
|
"high-risk-action",
|
||||||
|
"policy-risk",
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const reason of reasons) {
|
||||||
|
const decision = decide({
|
||||||
|
classified: { confidence: 0.8 },
|
||||||
|
reason,
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "review",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(decision).toMatchObject({
|
||||||
|
reason,
|
||||||
|
shouldHandoff: true,
|
||||||
|
handoff: {
|
||||||
|
reason,
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "review",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("escalates exhausted budgets and validation rejects", () => {
|
||||||
|
expect(
|
||||||
|
decide({
|
||||||
|
budget: {
|
||||||
|
estimatedCostUsd: 0,
|
||||||
|
handoffs: 4,
|
||||||
|
maxCostUsd: 0.5,
|
||||||
|
maxHandoffs: 4,
|
||||||
|
maxSameAgentVisits: 2,
|
||||||
|
maxToolCalls: 20,
|
||||||
|
toolCalls: 0,
|
||||||
|
visits: {},
|
||||||
|
},
|
||||||
|
}).handoff?.reason,
|
||||||
|
).toBe("max-hop");
|
||||||
|
|
||||||
|
expect(decide({ validation: { status: "reject" } }).handoff?.reason).toBe(
|
||||||
|
"validation-failed",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("escalates low-confidence and uncertain high-risk responses", () => {
|
||||||
|
expect(
|
||||||
|
decide({ classified: { confidence: 0.2 } }).handoff?.reason,
|
||||||
|
).toBe("low-confidence");
|
||||||
|
|
||||||
|
expect(
|
||||||
|
decide({
|
||||||
|
capabilityCard: {
|
||||||
|
canHandle: ["policy"],
|
||||||
|
cannotHandle: [],
|
||||||
|
domain: "moderation",
|
||||||
|
handoffTargets: ["supervisor"],
|
||||||
|
outputContract: { format: "text", maxChars: "500" },
|
||||||
|
riskTier: "high",
|
||||||
|
},
|
||||||
|
classified: { confidence: 0.6 },
|
||||||
|
}).handoff?.reason,
|
||||||
|
).toBe("high-risk-action");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not escalate ordinary confident responses", () => {
|
||||||
|
expect(
|
||||||
|
decide({
|
||||||
|
capabilityCard: {
|
||||||
|
canHandle: ["help"],
|
||||||
|
cannotHandle: [],
|
||||||
|
domain: "onboarding",
|
||||||
|
handoffTargets: ["supervisor"],
|
||||||
|
outputContract: { format: "text", maxChars: "500" },
|
||||||
|
riskTier: "medium",
|
||||||
|
},
|
||||||
|
classified: { confidence: 0.9 },
|
||||||
|
}),
|
||||||
|
).toEqual({ shouldHandoff: false });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import type { HandoffReason, HandoffRequest } from "@agent/types";
|
||||||
|
import type { BudgetSnapshot } from "@agent/budget";
|
||||||
|
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||||
|
|
||||||
|
export interface EscalationDecision {
|
||||||
|
handoff?: HandoffRequest | undefined;
|
||||||
|
reason?: HandoffReason | undefined;
|
||||||
|
shouldHandoff: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function decide(input: {
|
||||||
|
budget?: BudgetSnapshot | undefined;
|
||||||
|
capabilityCard?: SkillCapabilityCard | undefined;
|
||||||
|
classified?: { confidence?: number | undefined } | undefined;
|
||||||
|
reason?: HandoffReason | undefined;
|
||||||
|
targetAgent?: string | undefined;
|
||||||
|
taskSpec?: string | undefined;
|
||||||
|
validation?: { status?: string | undefined } | undefined;
|
||||||
|
}): EscalationDecision {
|
||||||
|
const confidence = input.classified?.confidence;
|
||||||
|
let reason: HandoffReason | undefined = input.reason;
|
||||||
|
|
||||||
|
if (!reason) {
|
||||||
|
if (input.budget && input.budget.handoffs >= input.budget.maxHandoffs) {
|
||||||
|
reason = "max-hop";
|
||||||
|
} else if (input.validation?.status === "reject") {
|
||||||
|
reason = "validation-failed";
|
||||||
|
} else if (typeof confidence === "number" && confidence < 0.35) {
|
||||||
|
reason = "low-confidence";
|
||||||
|
} else if (
|
||||||
|
input.capabilityCard?.riskTier === "high" &&
|
||||||
|
typeof confidence === "number" &&
|
||||||
|
confidence < 0.7
|
||||||
|
) {
|
||||||
|
reason = "high-risk-action";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!reason) {
|
||||||
|
return { shouldHandoff: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
handoff: {
|
||||||
|
confidence: confidence ?? 0.5,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason,
|
||||||
|
risk: input.capabilityCard?.riskTier ?? "medium",
|
||||||
|
targetAgent: input.targetAgent ?? "supervisor",
|
||||||
|
taskSpec: input.taskSpec ?? "Escalate this request to the appropriate skill.",
|
||||||
|
},
|
||||||
|
reason,
|
||||||
|
shouldHandoff: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -3,6 +3,15 @@ name: moderation
|
|||||||
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
|
description: 커뮤니티 규칙 문의와 위반 안내를 담당하는 PoC skill.
|
||||||
version: 1
|
version: 1
|
||||||
priority: 50
|
priority: 50
|
||||||
|
domain: moderation
|
||||||
|
canHandle: [rules, violation-report, appeal, sanctions, policy-risk]
|
||||||
|
cannotHandle: [onboarding-status, nickname-format, photo-verification, discord-onboarding]
|
||||||
|
handoffTargets: [onboarding, supervisor]
|
||||||
|
riskTier: high
|
||||||
|
outputContract:
|
||||||
|
format: text
|
||||||
|
maxChars: "500"
|
||||||
|
requiredFields: text
|
||||||
intents: [greeting, rule-question, violation-report, appeal, off-topic]
|
intents: [greeting, rule-question, violation-report, appeal, off-topic]
|
||||||
triggers: [user-message]
|
triggers: [user-message]
|
||||||
tools: [lookup-rule]
|
tools: [lookup-rule]
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import type {
|
|||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import type { ChatMessage } from "@llm/types";
|
import type { ChatMessage } from "@llm/types";
|
||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
|
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||||
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
import { lookupRuleTool } from "@tools/moderation/lookup-rule";
|
||||||
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
import { classifyModerationIntent, type ModerationIntent } from "./intents.js";
|
||||||
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
import { MODERATION_PROMPT_DATA } from "./prompt-data.js";
|
||||||
@@ -22,6 +23,24 @@ type RuleTopic =
|
|||||||
| "sanctions"
|
| "sanctions"
|
||||||
| "spam";
|
| "spam";
|
||||||
|
|
||||||
|
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||||
|
canHandle: ["rules", "violation-report", "appeal", "sanctions", "policy-risk"],
|
||||||
|
cannotHandle: [
|
||||||
|
"onboarding-status",
|
||||||
|
"nickname-format",
|
||||||
|
"photo-verification",
|
||||||
|
"discord-onboarding",
|
||||||
|
],
|
||||||
|
domain: "moderation",
|
||||||
|
handoffTargets: ["onboarding", "supervisor"],
|
||||||
|
outputContract: {
|
||||||
|
format: "text",
|
||||||
|
maxChars: "500",
|
||||||
|
requiredFields: "text",
|
||||||
|
},
|
||||||
|
riskTier: "high",
|
||||||
|
};
|
||||||
|
|
||||||
function formatHistory(history: ConversationMessage[]): string {
|
function formatHistory(history: ConversationMessage[]): string {
|
||||||
if (history.length === 0) {
|
if (history.length === 0) {
|
||||||
return "(none)";
|
return "(none)";
|
||||||
@@ -156,6 +175,10 @@ export class ModerationSkill implements Skill {
|
|||||||
return request.skill === this.name;
|
return request.skill === this.name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
capabilityCard(): SkillCapabilityCard {
|
||||||
|
return CAPABILITY_CARD;
|
||||||
|
}
|
||||||
|
|
||||||
async handle(
|
async handle(
|
||||||
request: AgentRequest,
|
request: AgentRequest,
|
||||||
conversationHistory: ConversationMessage[],
|
conversationHistory: ConversationMessage[],
|
||||||
|
|||||||
@@ -3,6 +3,15 @@ name: onboarding
|
|||||||
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
|
description: 카카오톡 오픈채팅 새내기 등록을 도와주는 친절한 선배 역할.
|
||||||
version: 1
|
version: 1
|
||||||
priority: 70
|
priority: 70
|
||||||
|
domain: onboarding
|
||||||
|
canHandle: [new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed, onboarding-help]
|
||||||
|
cannotHandle: [moderation-report, sanctions, appeal, policy-enforcement]
|
||||||
|
handoffTargets: [moderation, supervisor]
|
||||||
|
riskTier: medium
|
||||||
|
outputContract:
|
||||||
|
format: text
|
||||||
|
maxChars: "500"
|
||||||
|
requiredFields: text
|
||||||
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
|
intents: [greeting, acknowledgement, status-question, help-how, help-where, nickname-question, photo-question, discord-question, discord-verify-question, age-question]
|
||||||
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
|
triggers: [user-message, new-member, nickname-invalid, photo-failed, progress-update, timeout-warning, completed]
|
||||||
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
|
tools: [validate-nickname, check-onboarding-progress, lookup-community-rule, get-room-context, verify-discord-nickname]
|
||||||
|
|||||||
@@ -79,4 +79,14 @@ describe("validateOnboardingResponse", () => {
|
|||||||
|
|
||||||
expect(result.status).toBe("pass");
|
expect(result.status).toBe("pass");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("passes handoff control tokens", () => {
|
||||||
|
const result = validateOnboardingResponse(
|
||||||
|
"[HANDOFF:moderation:policy-risk]",
|
||||||
|
makeDerived(),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.status).toBe("pass");
|
||||||
|
expect(result.reasons).toContain("handoff");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { trace } from "@opentelemetry/api";
|
import { trace } from "@opentelemetry/api";
|
||||||
import { getMaxResponseLength } from "../../../config.js";
|
import { getMaxResponseLength } from "../../../config.js";
|
||||||
|
import { parseControlTokens } from "../../control-token-parser.js";
|
||||||
import { VALID_TIER_SET } from "../constants.js";
|
import { VALID_TIER_SET } from "../constants.js";
|
||||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||||
import type { DerivedState } from "../state/onboarding-state.js";
|
import type { DerivedState } from "../state/onboarding-state.js";
|
||||||
@@ -131,15 +132,12 @@ export function validateOnboardingResponse(
|
|||||||
derived: DerivedState,
|
derived: DerivedState,
|
||||||
options?: { allowShortResponse?: boolean | undefined },
|
options?: { allowShortResponse?: boolean | undefined },
|
||||||
): ValidationResult {
|
): ValidationResult {
|
||||||
const text = postProcessResponse(draft);
|
const parsedTokens = parseControlTokens(postProcessResponse(draft));
|
||||||
|
const text = parsedTokens.stripped;
|
||||||
const rejectReasons: string[] = [];
|
const rejectReasons: string[] = [];
|
||||||
const softWarnReasons: string[] = [];
|
const softWarnReasons: string[] = [];
|
||||||
|
|
||||||
if (!text) {
|
if (parsedTokens.tokens.some((token) => token.kind === "no-response")) {
|
||||||
rejectReasons.push("empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (text.includes("[NO_RESPONSE]")) {
|
|
||||||
return markValidation({
|
return markValidation({
|
||||||
status: "pass",
|
status: "pass",
|
||||||
text: "[NO_RESPONSE]",
|
text: "[NO_RESPONSE]",
|
||||||
@@ -147,6 +145,18 @@ export function validateOnboardingResponse(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (parsedTokens.tokens.some((token) => token.kind === "handoff")) {
|
||||||
|
return markValidation({
|
||||||
|
status: "pass",
|
||||||
|
text,
|
||||||
|
reasons: ["handoff"],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!text) {
|
||||||
|
rejectReasons.push("empty");
|
||||||
|
}
|
||||||
|
|
||||||
if (!options?.allowShortResponse && text.length < 10) {
|
if (!options?.allowShortResponse && text.length < 10) {
|
||||||
rejectReasons.push("minLength");
|
rejectReasons.push("minLength");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { getContext, getRequestLogger } from '@obs/context';
|
|||||||
import type { LlmClient } from '@llm/llm-client';
|
import type { LlmClient } from '@llm/llm-client';
|
||||||
import type { RoomContextService } from '@room/room-context-service';
|
import type { RoomContextService } from '@room/room-context-service';
|
||||||
import type { Skill } from '@skills/skill';
|
import type { Skill } from '@skills/skill';
|
||||||
|
import type { SkillCapabilityCard } from '@skills/skill-loader';
|
||||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||||
import { createOnboardingToolRegistry } from './tools.js';
|
import { createOnboardingToolRegistry } from './tools.js';
|
||||||
@@ -68,6 +69,32 @@ import type {
|
|||||||
StoredOnboardingSession,
|
StoredOnboardingSession,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
|
|
||||||
|
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||||
|
canHandle: [
|
||||||
|
'new-member',
|
||||||
|
'nickname-invalid',
|
||||||
|
'photo-failed',
|
||||||
|
'progress-update',
|
||||||
|
'timeout-warning',
|
||||||
|
'completed',
|
||||||
|
'onboarding-help',
|
||||||
|
],
|
||||||
|
cannotHandle: [
|
||||||
|
'moderation-report',
|
||||||
|
'sanctions',
|
||||||
|
'appeal',
|
||||||
|
'policy-enforcement',
|
||||||
|
],
|
||||||
|
domain: 'onboarding',
|
||||||
|
handoffTargets: ['moderation', 'supervisor'],
|
||||||
|
outputContract: {
|
||||||
|
format: 'text',
|
||||||
|
maxChars: '500',
|
||||||
|
requiredFields: 'text',
|
||||||
|
},
|
||||||
|
riskTier: 'medium',
|
||||||
|
};
|
||||||
|
|
||||||
export class OnboardingSkill implements Skill {
|
export class OnboardingSkill implements Skill {
|
||||||
readonly name = 'onboarding';
|
readonly name = 'onboarding';
|
||||||
readonly priority = 70;
|
readonly priority = 70;
|
||||||
@@ -99,6 +126,10 @@ export class OnboardingSkill implements Skill {
|
|||||||
return request.skill === 'onboarding';
|
return request.skill === 'onboarding';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
capabilityCard(): SkillCapabilityCard {
|
||||||
|
return CAPABILITY_CARD;
|
||||||
|
}
|
||||||
|
|
||||||
async handle(
|
async handle(
|
||||||
request: AgentRequest,
|
request: AgentRequest,
|
||||||
conversationHistory: ConversationMessage[],
|
conversationHistory: ConversationMessage[],
|
||||||
|
|||||||
@@ -21,6 +21,15 @@ function makeTempSkillRoot(): string {
|
|||||||
"priority: 65",
|
"priority: 65",
|
||||||
"intents: [hello, help]",
|
"intents: [hello, help]",
|
||||||
"triggers: [user-message]",
|
"triggers: [user-message]",
|
||||||
|
"domain: demo-domain",
|
||||||
|
"canHandle: [demo-task]",
|
||||||
|
"cannotHandle: [other-task]",
|
||||||
|
"handoffTargets: [supervisor]",
|
||||||
|
"riskTier: medium",
|
||||||
|
"outputContract:",
|
||||||
|
" format: text",
|
||||||
|
" maxChars: \"500\"",
|
||||||
|
" requiredFields: text",
|
||||||
"tools: [demo-tool]",
|
"tools: [demo-tool]",
|
||||||
"resources:",
|
"resources:",
|
||||||
" guide: ./resources/guide.md",
|
" guide: ./resources/guide.md",
|
||||||
@@ -41,8 +50,18 @@ describe("SkillRegistry", () => {
|
|||||||
expect(manifest).toMatchObject({
|
expect(manifest).toMatchObject({
|
||||||
name: "demo",
|
name: "demo",
|
||||||
description: "Demo skill.",
|
description: "Demo skill.",
|
||||||
|
canHandle: ["demo-task"],
|
||||||
|
cannotHandle: ["other-task"],
|
||||||
|
domain: "demo-domain",
|
||||||
|
handoffTargets: ["supervisor"],
|
||||||
intents: ["hello", "help"],
|
intents: ["hello", "help"],
|
||||||
|
outputContract: {
|
||||||
|
format: "text",
|
||||||
|
maxChars: "500",
|
||||||
|
requiredFields: "text",
|
||||||
|
},
|
||||||
priority: 65,
|
priority: 65,
|
||||||
|
riskTier: "medium",
|
||||||
tools: ["demo-tool"],
|
tools: ["demo-tool"],
|
||||||
trustTier: "T2",
|
trustTier: "T2",
|
||||||
version: 1,
|
version: 1,
|
||||||
|
|||||||
@@ -3,14 +3,36 @@ import { dirname, resolve } from "node:path";
|
|||||||
import { getLogger } from "@obs/logger";
|
import { getLogger } from "@obs/logger";
|
||||||
|
|
||||||
export type TrustTier = "T1" | "T2" | "T3" | "T4";
|
export type TrustTier = "T1" | "T2" | "T3" | "T4";
|
||||||
|
export type SkillRiskTier = "low" | "medium" | "high";
|
||||||
|
|
||||||
|
export interface SkillOutputContract {
|
||||||
|
format?: string | undefined;
|
||||||
|
maxChars?: string | undefined;
|
||||||
|
requiredFields?: string | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SkillCapabilityCard {
|
||||||
|
canHandle: string[];
|
||||||
|
cannotHandle: string[];
|
||||||
|
domain: string;
|
||||||
|
handoffTargets: string[];
|
||||||
|
outputContract: SkillOutputContract;
|
||||||
|
riskTier: SkillRiskTier;
|
||||||
|
}
|
||||||
|
|
||||||
export interface SkillManifest {
|
export interface SkillManifest {
|
||||||
bodyPath: string;
|
bodyPath: string;
|
||||||
|
canHandle: string[];
|
||||||
|
cannotHandle: string[];
|
||||||
description: string;
|
description: string;
|
||||||
|
domain: string;
|
||||||
|
handoffTargets: string[];
|
||||||
intents: string[];
|
intents: string[];
|
||||||
name: string;
|
name: string;
|
||||||
|
outputContract: SkillOutputContract;
|
||||||
priority: number;
|
priority: number;
|
||||||
resources: Record<string, string>;
|
resources: Record<string, string>;
|
||||||
|
riskTier: SkillRiskTier;
|
||||||
tools: string[];
|
tools: string[];
|
||||||
triggers: string[];
|
triggers: string[];
|
||||||
trustTier: TrustTier;
|
trustTier: TrustTier;
|
||||||
@@ -121,10 +143,32 @@ function readResources(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readStringMap(
|
||||||
|
fields: Record<string, unknown>,
|
||||||
|
key: string,
|
||||||
|
): Record<string, string> {
|
||||||
|
const raw = fields[key];
|
||||||
|
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(raw as Record<string, unknown>).flatMap(([mapKey, value]) =>
|
||||||
|
typeof value === "string" && value.trim()
|
||||||
|
? [[mapKey, value.trim().replace(/^["']|["']$/gu, "")]]
|
||||||
|
: [],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function isTrustTier(value: unknown): value is TrustTier {
|
function isTrustTier(value: unknown): value is TrustTier {
|
||||||
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
|
return value === "T1" || value === "T2" || value === "T3" || value === "T4";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isRiskTier(value: unknown): value is SkillRiskTier {
|
||||||
|
return value === "low" || value === "medium" || value === "high";
|
||||||
|
}
|
||||||
|
|
||||||
function isPriority(value: unknown): value is number {
|
function isPriority(value: unknown): value is number {
|
||||||
return (
|
return (
|
||||||
typeof value === "number" &&
|
typeof value === "number" &&
|
||||||
@@ -147,6 +191,8 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
|||||||
const priority = parsed.fields["priority"] ?? 50;
|
const priority = parsed.fields["priority"] ?? 50;
|
||||||
const version = parsed.fields["version"];
|
const version = parsed.fields["version"];
|
||||||
const trustTier = parsed.fields["trustTier"];
|
const trustTier = parsed.fields["trustTier"];
|
||||||
|
const domain = parsed.fields["domain"];
|
||||||
|
const riskTier = parsed.fields["riskTier"] ?? "low";
|
||||||
|
|
||||||
if (
|
if (
|
||||||
typeof name !== "string" ||
|
typeof name !== "string" ||
|
||||||
@@ -155,20 +201,28 @@ function parseSkillFile(path: string): ParsedSkillFile | null {
|
|||||||
!description.trim() ||
|
!description.trim() ||
|
||||||
!isPriority(priority) ||
|
!isPriority(priority) ||
|
||||||
typeof version !== "number" ||
|
typeof version !== "number" ||
|
||||||
!isTrustTier(trustTier)
|
!isTrustTier(trustTier) ||
|
||||||
|
!isRiskTier(riskTier)
|
||||||
) {
|
) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
const outputContract = readStringMap(parsed.fields, "outputContract");
|
||||||
|
|
||||||
return {
|
return {
|
||||||
body: parsed.body,
|
body: parsed.body,
|
||||||
manifest: {
|
manifest: {
|
||||||
bodyPath: path,
|
bodyPath: path,
|
||||||
|
canHandle: readStringArray(parsed.fields, "canHandle"),
|
||||||
|
cannotHandle: readStringArray(parsed.fields, "cannotHandle"),
|
||||||
description,
|
description,
|
||||||
|
domain: typeof domain === "string" && domain.trim() ? domain : "general",
|
||||||
|
handoffTargets: readStringArray(parsed.fields, "handoffTargets"),
|
||||||
intents: readStringArray(parsed.fields, "intents"),
|
intents: readStringArray(parsed.fields, "intents"),
|
||||||
name,
|
name,
|
||||||
|
outputContract,
|
||||||
priority,
|
priority,
|
||||||
resources: readResources(parsed.fields, skillDir),
|
resources: readResources(parsed.fields, skillDir),
|
||||||
|
riskTier,
|
||||||
tools: readStringArray(parsed.fields, "tools"),
|
tools: readStringArray(parsed.fields, "tools"),
|
||||||
triggers: readStringArray(parsed.fields, "triggers"),
|
triggers: readStringArray(parsed.fields, "triggers"),
|
||||||
trustTier,
|
trustTier,
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ import type {
|
|||||||
AgentResponse,
|
AgentResponse,
|
||||||
ConversationMessage,
|
ConversationMessage,
|
||||||
} from "@agent/types";
|
} from "@agent/types";
|
||||||
|
import type { SkillCapabilityCard } from "./skill-loader.js";
|
||||||
|
|
||||||
export interface Skill {
|
export interface Skill {
|
||||||
readonly name: string;
|
readonly name: string;
|
||||||
readonly priority: number;
|
readonly priority: number;
|
||||||
|
capabilityCard?(): SkillCapabilityCard;
|
||||||
canHandle(request: AgentRequest): boolean;
|
canHandle(request: AgentRequest): boolean;
|
||||||
handle(
|
handle(
|
||||||
request: AgentRequest,
|
request: AgentRequest,
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
---
|
||||||
|
name: supervisor
|
||||||
|
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
|
||||||
|
version: 1
|
||||||
|
priority: 10
|
||||||
|
domain: meta
|
||||||
|
canHandle: [*]
|
||||||
|
cannotHandle: []
|
||||||
|
handoffTargets: [onboarding, moderation]
|
||||||
|
riskTier: medium
|
||||||
|
outputContract:
|
||||||
|
format: text
|
||||||
|
maxChars: "500"
|
||||||
|
requiredFields: text
|
||||||
|
intents: []
|
||||||
|
triggers: [handoff, user-message]
|
||||||
|
tools: []
|
||||||
|
resources:
|
||||||
|
trustTier: T4
|
||||||
|
---
|
||||||
|
|
||||||
|
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import type { Skill } from "@skills/skill";
|
||||||
|
import type { SkillDeps } from "@skills/skill-factory";
|
||||||
|
import { SupervisorSkill } from "./skill.js";
|
||||||
|
|
||||||
|
export { SupervisorSkill } from "./skill.js";
|
||||||
|
|
||||||
|
export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||||
|
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
export function getIntentKinds(): string[] {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getToolNames(): string[] {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import type { GarnetClient } from "@common/garnet";
|
||||||
|
|
||||||
|
const SUPERVISOR_REVIEW_STREAM = "lolympus-agent:review-queue";
|
||||||
|
|
||||||
|
export async function appendSupervisorReview(
|
||||||
|
garnetClient: Pick<GarnetClient, "appendStream">,
|
||||||
|
fields: Record<string, string>,
|
||||||
|
): Promise<void> {
|
||||||
|
await garnetClient.appendStream(SUPERVISOR_REVIEW_STREAM, {
|
||||||
|
...fields,
|
||||||
|
type: "supervisor-handoff",
|
||||||
|
queuedAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||||
|
|
||||||
|
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
|
||||||
|
const text = [
|
||||||
|
packet?.userGoal,
|
||||||
|
packet?.summary,
|
||||||
|
packet?.handoffReason,
|
||||||
|
].join(" ");
|
||||||
|
|
||||||
|
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) {
|
||||||
|
return "moderation";
|
||||||
|
}
|
||||||
|
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) {
|
||||||
|
return "onboarding";
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import type { AlertPayload } from "@common/nats-subjects";
|
||||||
|
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||||
|
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||||
|
import { SupervisorSkill } from "./skill.js";
|
||||||
|
|
||||||
|
class FakeGarnet {
|
||||||
|
readonly streams: Array<{ key: string; fields: Record<string, unknown> }> = [];
|
||||||
|
|
||||||
|
appendStream(key: string, fields: Record<string, unknown>): Promise<string> {
|
||||||
|
this.streams.push({ key, fields });
|
||||||
|
return Promise.resolve("1-0");
|
||||||
|
}
|
||||||
|
|
||||||
|
setStringIfAbsent(): Promise<boolean> {
|
||||||
|
return Promise.resolve(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeNats {
|
||||||
|
readonly published: AlertPayload[] = [];
|
||||||
|
|
||||||
|
publish<T = unknown>(_subject: string, payload: T): Promise<void> {
|
||||||
|
this.published.push(payload as AlertPayload);
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseRequest: AgentRequest = {
|
||||||
|
chatroomId: "room-1",
|
||||||
|
nickname: "tester",
|
||||||
|
requestId: "request-1",
|
||||||
|
skill: "supervisor",
|
||||||
|
trigger: "handoff",
|
||||||
|
userId: "user-1",
|
||||||
|
};
|
||||||
|
|
||||||
|
const sourceResponse: AgentResponse = {
|
||||||
|
confidence: 0.3,
|
||||||
|
requestId: "request-1",
|
||||||
|
source: "template",
|
||||||
|
text: "needs review",
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("SupervisorSkill", () => {
|
||||||
|
it("queues review and publishes alert for handoffs", async () => {
|
||||||
|
const garnet = new FakeGarnet();
|
||||||
|
const nats = new FakeNats();
|
||||||
|
const skill = new SupervisorSkill(garnet, nats);
|
||||||
|
const packet = buildHandoffPacket({
|
||||||
|
handoff: {
|
||||||
|
confidence: 0.3,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "max-hop",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "가입 상태 확인",
|
||||||
|
},
|
||||||
|
request: baseRequest,
|
||||||
|
response: sourceResponse,
|
||||||
|
sourceChain: ["onboarding", "supervisor"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await skill.handle(
|
||||||
|
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.text).toContain("운영자");
|
||||||
|
expect(response.metadata).toMatchObject({
|
||||||
|
supervisorReason: "max-hop",
|
||||||
|
suggestedTarget: "onboarding",
|
||||||
|
});
|
||||||
|
expect(garnet.streams[0]).toMatchObject({
|
||||||
|
key: "lolympus-agent:review-queue",
|
||||||
|
fields: {
|
||||||
|
reason: "max-hop",
|
||||||
|
requestId: "request-1",
|
||||||
|
target: "onboarding",
|
||||||
|
type: "supervisor-handoff",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(nats.published[0]).toMatchObject({
|
||||||
|
kind: "supervisor-handoff",
|
||||||
|
metric: "agent.supervisor_handoff",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import type {
|
||||||
|
AgentRequest,
|
||||||
|
AgentResponse,
|
||||||
|
ConversationMessage,
|
||||||
|
} from "@agent/types";
|
||||||
|
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||||
|
import type { GarnetClient } from "@common/garnet";
|
||||||
|
import type { NatsClient } from "@common/nats";
|
||||||
|
import { AlertService } from "@obs/alert";
|
||||||
|
import { metrics } from "@obs/metrics";
|
||||||
|
import type { Skill } from "@skills/skill";
|
||||||
|
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||||
|
import { appendSupervisorReview } from "./review-queue.js";
|
||||||
|
import { routeSupervisorTarget } from "./routing-rules.js";
|
||||||
|
|
||||||
|
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||||
|
canHandle: ["*"],
|
||||||
|
cannotHandle: [],
|
||||||
|
domain: "meta",
|
||||||
|
handoffTargets: ["onboarding", "moderation"],
|
||||||
|
outputContract: {
|
||||||
|
format: "text",
|
||||||
|
maxChars: "500",
|
||||||
|
requiredFields: "text",
|
||||||
|
},
|
||||||
|
riskTier: "medium",
|
||||||
|
};
|
||||||
|
|
||||||
|
export class SupervisorSkill implements Skill {
|
||||||
|
readonly name = "supervisor";
|
||||||
|
readonly priority = 10;
|
||||||
|
private readonly alertService: AlertService;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||||
|
natsClient: Pick<NatsClient, "publish">,
|
||||||
|
) {
|
||||||
|
this.alertService = new AlertService(garnetClient, natsClient);
|
||||||
|
}
|
||||||
|
|
||||||
|
canHandle(request: AgentRequest): boolean {
|
||||||
|
return request.skill === this.name;
|
||||||
|
}
|
||||||
|
|
||||||
|
capabilityCard(): SkillCapabilityCard {
|
||||||
|
return CAPABILITY_CARD;
|
||||||
|
}
|
||||||
|
|
||||||
|
async handle(
|
||||||
|
request: AgentRequest,
|
||||||
|
_conversationHistory: ConversationMessage[],
|
||||||
|
): Promise<AgentResponse> {
|
||||||
|
const packet = readHandoffPacket(request);
|
||||||
|
const target = routeSupervisorTarget(packet);
|
||||||
|
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||||
|
|
||||||
|
await appendSupervisorReview(this.garnetClient, {
|
||||||
|
chatroomId: request.chatroomId,
|
||||||
|
reason,
|
||||||
|
requestId: request.requestId,
|
||||||
|
target: target ?? "human-review",
|
||||||
|
userId: request.userId,
|
||||||
|
});
|
||||||
|
metrics.humanOverrideTotal.inc({ kind: reason });
|
||||||
|
|
||||||
|
await this.alertService.sendAlert(
|
||||||
|
{
|
||||||
|
kind: "supervisor-handoff",
|
||||||
|
metric: "agent.supervisor_handoff",
|
||||||
|
severity: "warn",
|
||||||
|
suggestedAction: "Review the supervisor handoff queue.",
|
||||||
|
threshold: 0,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
value: 1,
|
||||||
|
window: "request",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
dedupeKey: `lolympus-agent:alert:dedupe:supervisor:${request.requestId}`,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
confidence: 0.6,
|
||||||
|
metadata: {
|
||||||
|
supervisorReason: reason,
|
||||||
|
suggestedTarget: target,
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { SkillRegistry } from "./skill-loader.js";
|
||||||
|
import { TransitionMatrix } from "./transition-matrix.js";
|
||||||
|
|
||||||
|
function writeSkill(
|
||||||
|
root: string,
|
||||||
|
name: string,
|
||||||
|
handoffTargets: string[],
|
||||||
|
): void {
|
||||||
|
const skillDir = join(root, name);
|
||||||
|
mkdirSync(skillDir, { recursive: true });
|
||||||
|
writeFileSync(
|
||||||
|
join(skillDir, "SKILL.md"),
|
||||||
|
[
|
||||||
|
"---",
|
||||||
|
`name: ${name}`,
|
||||||
|
`description: ${name} skill.`,
|
||||||
|
"version: 1",
|
||||||
|
"priority: 50",
|
||||||
|
"domain: test",
|
||||||
|
"canHandle: [test]",
|
||||||
|
"cannotHandle: []",
|
||||||
|
`handoffTargets: [${handoffTargets.join(", ")}]`,
|
||||||
|
"riskTier: low",
|
||||||
|
"outputContract:",
|
||||||
|
" format: text",
|
||||||
|
" maxChars: \"500\"",
|
||||||
|
"trustTier: T2",
|
||||||
|
"---",
|
||||||
|
"",
|
||||||
|
"body",
|
||||||
|
].join("\n"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("TransitionMatrix", () => {
|
||||||
|
it("allows declared handoffs and blocks self loops", () => {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-"));
|
||||||
|
writeSkill(root, "onboarding", ["moderation"]);
|
||||||
|
writeSkill(root, "moderation", []);
|
||||||
|
|
||||||
|
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
|
||||||
|
|
||||||
|
expect(matrix.isAllowed("onboarding", "moderation")).toBe(true);
|
||||||
|
expect(matrix.isAllowed("onboarding", "supervisor")).toBe(true);
|
||||||
|
expect(matrix.isAllowed("onboarding", "onboarding")).toBe(false);
|
||||||
|
expect(matrix.isAllowed("moderation", "onboarding")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports unknown handoff targets", () => {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "lolympus-transition-bad-"));
|
||||||
|
writeSkill(root, "onboarding", ["missing"]);
|
||||||
|
|
||||||
|
const matrix = TransitionMatrix.fromRegistry(SkillRegistry.load(root));
|
||||||
|
|
||||||
|
expect(matrix.validateGraph(["onboarding"])).toContain(
|
||||||
|
"onboarding: unknown handoff target missing",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
import type { SkillRegistry } from "./skill-loader.js";
|
||||||
|
|
||||||
|
export class TransitionMatrix {
|
||||||
|
private constructor(private readonly transitions: Map<string, Set<string>>) {}
|
||||||
|
|
||||||
|
static fromRegistry(registry: SkillRegistry): TransitionMatrix {
|
||||||
|
const transitions = new Map<string, Set<string>>();
|
||||||
|
for (const manifest of registry.list()) {
|
||||||
|
const targets =
|
||||||
|
manifest.name === "supervisor"
|
||||||
|
? manifest.handoffTargets
|
||||||
|
: [...manifest.handoffTargets, "supervisor"];
|
||||||
|
transitions.set(
|
||||||
|
manifest.name,
|
||||||
|
new Set(targets),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return new TransitionMatrix(transitions);
|
||||||
|
}
|
||||||
|
|
||||||
|
isAllowed(from: string, to: string): boolean {
|
||||||
|
if (from === to) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return this.transitions.get(from)?.has(to) ?? true;
|
||||||
|
}
|
||||||
|
|
||||||
|
targetsOf(from: string): string[] {
|
||||||
|
return [...(this.transitions.get(from) ?? new Set<string>())];
|
||||||
|
}
|
||||||
|
|
||||||
|
validateGraph(knownSkills?: readonly string[]): string[] {
|
||||||
|
const known = new Set([...(knownSkills ?? this.transitions.keys()), "supervisor"]);
|
||||||
|
const errors: string[] = [];
|
||||||
|
for (const [from, targets] of this.transitions.entries()) {
|
||||||
|
if (targets.has(from)) {
|
||||||
|
errors.push(`${from}: handoffTargets must not include itself`);
|
||||||
|
}
|
||||||
|
for (const target of targets) {
|
||||||
|
if (!known.has(target)) {
|
||||||
|
errors.push(`${from}: unknown handoff target ${target}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,8 +3,10 @@ import { resolve } from "node:path";
|
|||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
|
import { SkillRegistry, type SkillManifest } from "./skill-loader.js";
|
||||||
|
import { TransitionMatrix } from "./transition-matrix.js";
|
||||||
import * as moderationBinding from "./moderation/manifest-binding.js";
|
import * as moderationBinding from "./moderation/manifest-binding.js";
|
||||||
import * as onboardingBinding from "./onboarding/manifest-binding.js";
|
import * as onboardingBinding from "./onboarding/manifest-binding.js";
|
||||||
|
import * as supervisorBinding from "./supervisor/manifest-binding.js";
|
||||||
|
|
||||||
interface ManifestBinding {
|
interface ManifestBinding {
|
||||||
getIntentKinds(): string[];
|
getIntentKinds(): string[];
|
||||||
@@ -14,6 +16,7 @@ interface ManifestBinding {
|
|||||||
const BINDINGS: Record<string, ManifestBinding> = {
|
const BINDINGS: Record<string, ManifestBinding> = {
|
||||||
moderation: moderationBinding,
|
moderation: moderationBinding,
|
||||||
onboarding: onboardingBinding,
|
onboarding: onboardingBinding,
|
||||||
|
supervisor: supervisorBinding,
|
||||||
};
|
};
|
||||||
|
|
||||||
function sortUnique(values: readonly string[]): string[] {
|
function sortUnique(values: readonly string[]): string[] {
|
||||||
@@ -32,12 +35,15 @@ function diff(left: readonly string[], right: readonly string[]): string[] {
|
|||||||
function validateManifest(
|
function validateManifest(
|
||||||
manifest: SkillManifest,
|
manifest: SkillManifest,
|
||||||
binding: ManifestBinding | undefined,
|
binding: ManifestBinding | undefined,
|
||||||
|
knownSkills: readonly string[],
|
||||||
): string[] {
|
): string[] {
|
||||||
if (!binding) {
|
if (!binding) {
|
||||||
return [`${manifest.name}: missing manifest binding`];
|
return [`${manifest.name}: missing manifest binding`];
|
||||||
}
|
}
|
||||||
|
|
||||||
const errors: string[] = [];
|
const errors: string[] = [];
|
||||||
|
const capabilityErrors = validateCapabilityCard(manifest, knownSkills);
|
||||||
|
errors.push(...capabilityErrors);
|
||||||
const manifestIntents = sortUnique(manifest.intents);
|
const manifestIntents = sortUnique(manifest.intents);
|
||||||
const codeIntents = sortUnique(binding.getIntentKinds());
|
const codeIntents = sortUnique(binding.getIntentKinds());
|
||||||
const unknownIntents = diff(manifestIntents, codeIntents);
|
const unknownIntents = diff(manifestIntents, codeIntents);
|
||||||
@@ -66,6 +72,45 @@ function validateManifest(
|
|||||||
return errors;
|
return errors;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validateCapabilityCard(
|
||||||
|
manifest: SkillManifest,
|
||||||
|
knownSkills: readonly string[],
|
||||||
|
): string[] {
|
||||||
|
const errors: string[] = [];
|
||||||
|
const canHandle = new Set(manifest.canHandle);
|
||||||
|
const overlap = manifest.cannotHandle.filter((item) => canHandle.has(item));
|
||||||
|
if (overlap.length > 0) {
|
||||||
|
errors.push(
|
||||||
|
`${manifest.name}: canHandle and cannotHandle overlap: ${format(overlap)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const riskTier: string = manifest.riskTier;
|
||||||
|
if (riskTier !== "low" && riskTier !== "medium" && riskTier !== "high") {
|
||||||
|
errors.push(`${manifest.name}: invalid riskTier ${riskTier}`);
|
||||||
|
}
|
||||||
|
const knownTargets = new Set([...knownSkills, "supervisor"]);
|
||||||
|
const missingTargets = manifest.handoffTargets.filter(
|
||||||
|
(target) => !knownTargets.has(target),
|
||||||
|
);
|
||||||
|
if (missingTargets.length > 0) {
|
||||||
|
errors.push(
|
||||||
|
`${manifest.name}: unknown handoffTargets: ${format(missingTargets)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (manifest.handoffTargets.includes(manifest.name)) {
|
||||||
|
errors.push(`${manifest.name}: handoffTargets must not include itself`);
|
||||||
|
}
|
||||||
|
if (!manifest.domain.trim()) {
|
||||||
|
errors.push(`${manifest.name}: domain is required`);
|
||||||
|
}
|
||||||
|
if (!manifest.outputContract.format || !manifest.outputContract.maxChars) {
|
||||||
|
errors.push(
|
||||||
|
`${manifest.name}: outputContract.format and outputContract.maxChars are required`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
||||||
const byPriority = new Map<number, string[]>();
|
const byPriority = new Map<number, string[]>();
|
||||||
for (const manifest of manifests) {
|
for (const manifest of manifests) {
|
||||||
@@ -84,11 +129,13 @@ function validatePriorityConflicts(manifests: SkillManifest[]): string[] {
|
|||||||
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
|
export function validateSkillManifests(rootDir = resolve(process.cwd(), "src", "skills")): string[] {
|
||||||
const registry = SkillRegistry.load(rootDir);
|
const registry = SkillRegistry.load(rootDir);
|
||||||
const manifests = registry.list();
|
const manifests = registry.list();
|
||||||
|
const knownSkills = manifests.map((manifest) => manifest.name);
|
||||||
return [
|
return [
|
||||||
...manifests.flatMap((manifest) =>
|
...manifests.flatMap((manifest) =>
|
||||||
validateManifest(manifest, BINDINGS[manifest.name]),
|
validateManifest(manifest, BINDINGS[manifest.name], knownSkills),
|
||||||
),
|
),
|
||||||
...validatePriorityConflicts(manifests),
|
...validatePriorityConflicts(manifests),
|
||||||
|
...TransitionMatrix.fromRegistry(registry).validateGraph(knownSkills),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user