Add bounded skill handoff graph

This commit is contained in:
dahyeon committed 2026-05-18 17:22:12 -07:00
1 parent 54bc295ead
commit c14adac162
42 files changed
+2243 -31

No files matched your search

+447 -19
View File
@@ -5,8 +5,22 @@ import { getRequestLogger, runWithContext } from "@obs/context";
import { getLogger } from "@obs/logger";
import { withSkillSpan, withSpan } from "@obs/spans";
import type { Skill } from "@skills/skill";
import type { AgentRequest, AgentResponse } from "./types.js";
import type { TransitionMatrix } from "@skills/transition-matrix";
import { parseControlTokens } from "@skills/control-token-parser";
import { decide as decideEscalation } from "@skills/escalation/trigger";
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
import { metrics } from "@obs/metrics";
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
import { buildHandoffPacket } from "./handoff-packet.js";
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
HandoffReason,
HandoffRequest,
} from "./types.js";
import {
getAgentRuntimeVerifierEnabled,
getAgentRequestTimeoutMs,
getRequestSubject,
getSafetyMarginMs,
@@ -43,6 +57,7 @@ export class Agent {
constructor(
private readonly natsClient: NatsClient,
private readonly conversationStore: ConversationStore,
private readonly transitionMatrix?: TransitionMatrix | undefined,
) {}
registerSkill(skill: Skill): void {
@@ -107,18 +122,28 @@ export class Agent {
"[Agent] Request received",
);
const skill = selectSkillForRequest(this.skills, request);
let skill = selectSkillForRequest(this.skills, request);
if (!skill) {
logger.info("[Agent] No skill found for request");
const response: AgentResponse = {
requestId: request.requestId,
text: "",
confidence: 0,
source: "template",
const supervisorRequest: AgentRequest = {
...request,
skill: "supervisor",
trigger: "handoff",
};
rootSpan.setAttribute("agent.source", "template");
await this.natsClient.respond(msg, response);
return;
skill = selectSkillForRequest(this.skills, supervisorRequest);
if (skill) {
request = supervisorRequest;
} else {
logger.info("[Agent] No skill found for request");
const response: AgentResponse = {
requestId: request.requestId,
text: "",
confidence: 0,
source: "template",
};
rootSpan.setAttribute("agent.source", "template");
await this.natsClient.respond(msg, response);
return;
}
}
try {
@@ -139,14 +164,17 @@ export class Agent {
);
}
const response = await withSkillSpan(
skill.name,
{
"agent.skill": skill.name,
"agent.trigger": request.trigger,
},
() => skill.handle(request, history),
);
const startedAt = Date.now();
const budget = new BudgetTracker();
budget.recordVisit(skill.name);
const response = await this.dispatch({
budget,
chain: [skill.name],
depth: 0,
history,
request,
skill,
});
if (response.text.trim()) {
await this.conversationStore.appendMessage(
@@ -161,6 +189,8 @@ export class Agent {
}
await this.natsClient.respond(msg, response);
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
rootSpan.setAttribute(
"agent.source",
response.text.trim() ? response.source : "empty",
@@ -200,4 +230,402 @@ export class Agent {
},
);
}
private async dispatch(input: {
budget: BudgetTracker;
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}): Promise<AgentResponse> {
const { budget, chain, depth, history, request, skill } = input;
const contextBase = {
chatroomId: request.chatroomId,
deadlineAt: new Date(
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
).toISOString(),
requestId: request.requestId,
skill: skill.name,
startedAt: new Date().toISOString(),
trigger: request.trigger,
userId: request.userId,
};
return runWithContext(
{
...contextBase,
budget: budget.snapshot(),
handoffChain: chain,
handoffDepth: depth,
},
async () => {
const response = await this.runSkillOnce({
chain,
depth,
history,
request,
skill,
});
const normalized = this.normalizeControlTokens(response, skill.name);
const verified = await this.applyRuntimeVerifier({
chain,
depth,
history,
request,
response: normalized,
skill,
});
const escalated = this.applyEscalationTrigger({
budget,
request,
response: verified,
skill,
});
const handoff = readHandoffRequest(escalated);
if (!handoff) {
return escalated;
}
const next = this.prepareHandoff({
budget,
chain,
currentSkill: skill.name,
depth,
handoff,
history,
request,
response: escalated,
});
if (!next) {
return this.makeSupervisorFallback(request, handoff.reason);
}
return this.dispatch(next);
},
);
}
private async runSkillOnce(input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}): Promise<AgentResponse> {
return withSkillSpan(
input.skill.name,
{
"agent.handoff_chain": input.chain.join(">"),
"agent.handoff_depth": input.depth,
"agent.skill": input.skill.name,
"agent.trigger": input.request.trigger,
},
() => input.skill.handle(input.request, input.history),
);
}
private normalizeControlTokens(
response: AgentResponse,
currentSkill: string,
): AgentResponse {
const parsed = parseControlTokens(response.text);
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
const metadata = { ...(response.metadata ?? {}) };
if (handoff?.kind === "handoff") {
metadata.handoffRequest = {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: handoff.reason,
risk: "medium",
targetAgent: handoff.targetAgent,
taskSpec: `Continue handling from ${currentSkill}`,
} satisfies HandoffRequest;
}
return {
...response,
metadata,
text: parsed.stripped,
};
}
private async applyRuntimeVerifier(input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
response: AgentResponse;
skill: Skill;
}): Promise<AgentResponse> {
if (!getAgentRuntimeVerifierEnabled()) {
return input.response;
}
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
if (!verifier) {
return input.response;
}
const verdict = await verifier({
request: input.request,
response: input.response,
skill: input.skill.name,
});
if (verdict === "pass") {
return input.response;
}
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
if (verdict === "retry" && input.depth === 0) {
const retryResponse = await this.runSkillOnce({
...input,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
runtimeVerifierRetry: true,
},
},
});
const normalizedRetry = this.normalizeControlTokens(
retryResponse,
input.skill.name,
);
if (readHandoffRequest(normalizedRetry)) {
return normalizedRetry;
}
const retryVerdict = await verifier({
request: input.request,
response: normalizedRetry,
skill: input.skill.name,
});
if (retryVerdict === "pass") {
return {
...normalizedRetry,
metadata: {
...(normalizedRetry.metadata ?? {}),
runtimeVerifier: "retry-pass",
},
};
}
metrics.validatorFailTotal.inc({
skill: input.skill.name,
stage: `retry-${retryVerdict}`,
});
return this.withRuntimeVerifierHandoff(normalizedRetry);
}
return this.withRuntimeVerifierHandoff(input.response);
}
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: "validation-failed",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "Runtime verifier rejected the response.",
} satisfies HandoffRequest,
},
};
}
private applyEscalationTrigger(input: {
budget: BudgetTracker;
request: AgentRequest;
response: AgentResponse;
skill: Skill;
}): AgentResponse {
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
return input.response;
}
const decision = decideEscalation({
budget: input.budget.snapshot(),
capabilityCard: input.skill.capabilityCard?.(),
classified: { confidence: input.response.confidence },
targetAgent: "supervisor",
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
});
if (!decision.handoff) {
return input.response;
}
return {
...input.response,
metadata: {
...(input.response.metadata ?? {}),
handoffRequest: decision.handoff,
},
};
}
private prepareHandoff(input: {
budget: BudgetTracker;
chain: string[];
currentSkill: string;
depth: number;
handoff: HandoffRequest;
history: ConversationMessage[];
request: AgentRequest;
response: AgentResponse;
}):
| {
budget: BudgetTracker;
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}
| null {
const target = this.resolveHandoffTarget(input);
const nextSkill = selectSkillForRequest(this.skills, {
...input.request,
skill: target.targetAgent,
trigger: "handoff",
});
if (!nextSkill) {
return null;
}
input.budget.recordHandoff(nextSkill.name);
metrics.handoffTotal.inc({
from: input.currentSkill,
reason: target.reason,
to: nextSkill.name,
});
const nextChain = [...input.chain, nextSkill.name];
const packet = buildHandoffPacket({
handoff: {
...input.handoff,
reason: target.reason,
targetAgent: nextSkill.name,
},
history: input.history,
request: input.request,
response: input.response,
sourceChain: nextChain,
state: {
source: input.currentSkill,
},
});
return {
budget: input.budget,
chain: nextChain,
depth: input.depth + 1,
history: input.history,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
handoffPacket: packet,
},
skill: nextSkill.name,
trigger: "handoff",
},
skill: nextSkill,
};
}
private resolveHandoffTarget(input: {
budget: BudgetTracker;
currentSkill: string;
handoff: HandoffRequest;
}): { reason: HandoffReason; targetAgent: string } {
const requestedTarget = input.handoff.targetAgent;
const allowed = this.transitionMatrix?.isAllowed(
input.currentSkill,
requestedTarget,
) ?? true;
if (!allowed) {
return {
reason: "policy-risk",
targetAgent: "supervisor",
};
}
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
if (verdict === "ok") {
return {
reason: input.handoff.reason,
targetAgent: requestedTarget,
};
}
if (verdict === "same-agent-loop") {
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
}
return {
reason: budgetVerdictToReason(verdict),
targetAgent: "supervisor",
};
}
private makeSupervisorFallback(
request: AgentRequest,
reason: HandoffReason,
): AgentResponse {
return {
confidence: 0.4,
metadata: {
supervisorFallbackReason: reason,
},
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
};
}
}
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
if (verdict === "same-agent-loop") {
return "same-agent-revisit";
}
if (verdict === "max-hops") {
return "max-hop";
}
return "policy-risk";
}
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
const value = response.metadata?.["handoffRequest"];
if (!value || typeof value !== "object") {
return null;
}
const handoff = value as Partial<HandoffRequest>;
if (
typeof handoff.targetAgent !== "string" ||
typeof handoff.reason !== "string" ||
typeof handoff.taskSpec !== "string" ||
typeof handoff.confidence !== "number" ||
typeof handoff.expectedOutput !== "string"
) {
return null;
}
return {
confidence: handoff.confidence,
evidenceRefs: Array.isArray(handoff.evidenceRefs)
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
: [],
expectedOutput: handoff.expectedOutput,
reason: handoff.reason,
risk:
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
? handoff.risk
: "medium",
targetAgent: handoff.targetAgent,
taskSpec: handoff.taskSpec,
...(handoff.userVisibleMessage
? { userVisibleMessage: handoff.userVisibleMessage }
: {}),
};
}