Add bounded skill handoff graph
This commit is contained in:
1 parent
54bc295ead
commit
c14adac162
42 files changed
+2243
-31
No files matched your search
+447
-19
@@ -5,8 +5,22 @@ import { getRequestLogger, runWithContext } from "@obs/context";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import { withSkillSpan, withSpan } from "@obs/spans";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { AgentRequest, AgentResponse } from "./types.js";
|
||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||
import { parseControlTokens } from "@skills/control-token-parser";
|
||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
|
||||
import { buildHandoffPacket } from "./handoff-packet.js";
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
HandoffRequest,
|
||||
} from "./types.js";
|
||||
import {
|
||||
getAgentRuntimeVerifierEnabled,
|
||||
getAgentRequestTimeoutMs,
|
||||
getRequestSubject,
|
||||
getSafetyMarginMs,
|
||||
@@ -43,6 +57,7 @@ export class Agent {
|
||||
constructor(
|
||||
private readonly natsClient: NatsClient,
|
||||
private readonly conversationStore: ConversationStore,
|
||||
private readonly transitionMatrix?: TransitionMatrix | undefined,
|
||||
) {}
|
||||
|
||||
registerSkill(skill: Skill): void {
|
||||
@@ -107,18 +122,28 @@ export class Agent {
|
||||
"[Agent] Request received",
|
||||
);
|
||||
|
||||
const skill = selectSkillForRequest(this.skills, request);
|
||||
let skill = selectSkillForRequest(this.skills, request);
|
||||
if (!skill) {
|
||||
logger.info("[Agent] No skill found for request");
|
||||
const response: AgentResponse = {
|
||||
requestId: request.requestId,
|
||||
text: "",
|
||||
confidence: 0,
|
||||
source: "template",
|
||||
const supervisorRequest: AgentRequest = {
|
||||
...request,
|
||||
skill: "supervisor",
|
||||
trigger: "handoff",
|
||||
};
|
||||
rootSpan.setAttribute("agent.source", "template");
|
||||
await this.natsClient.respond(msg, response);
|
||||
return;
|
||||
skill = selectSkillForRequest(this.skills, supervisorRequest);
|
||||
if (skill) {
|
||||
request = supervisorRequest;
|
||||
} else {
|
||||
logger.info("[Agent] No skill found for request");
|
||||
const response: AgentResponse = {
|
||||
requestId: request.requestId,
|
||||
text: "",
|
||||
confidence: 0,
|
||||
source: "template",
|
||||
};
|
||||
rootSpan.setAttribute("agent.source", "template");
|
||||
await this.natsClient.respond(msg, response);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -139,14 +164,17 @@ export class Agent {
|
||||
);
|
||||
}
|
||||
|
||||
const response = await withSkillSpan(
|
||||
skill.name,
|
||||
{
|
||||
"agent.skill": skill.name,
|
||||
"agent.trigger": request.trigger,
|
||||
},
|
||||
() => skill.handle(request, history),
|
||||
);
|
||||
const startedAt = Date.now();
|
||||
const budget = new BudgetTracker();
|
||||
budget.recordVisit(skill.name);
|
||||
const response = await this.dispatch({
|
||||
budget,
|
||||
chain: [skill.name],
|
||||
depth: 0,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
});
|
||||
|
||||
if (response.text.trim()) {
|
||||
await this.conversationStore.appendMessage(
|
||||
@@ -161,6 +189,8 @@ export class Agent {
|
||||
}
|
||||
|
||||
await this.natsClient.respond(msg, response);
|
||||
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
|
||||
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
|
||||
rootSpan.setAttribute(
|
||||
"agent.source",
|
||||
response.text.trim() ? response.source : "empty",
|
||||
@@ -200,4 +230,402 @@ export class Agent {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async dispatch(input: {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
const { budget, chain, depth, history, request, skill } = input;
|
||||
const contextBase = {
|
||||
chatroomId: request.chatroomId,
|
||||
deadlineAt: new Date(
|
||||
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
||||
).toISOString(),
|
||||
requestId: request.requestId,
|
||||
skill: skill.name,
|
||||
startedAt: new Date().toISOString(),
|
||||
trigger: request.trigger,
|
||||
userId: request.userId,
|
||||
};
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
...contextBase,
|
||||
budget: budget.snapshot(),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
},
|
||||
async () => {
|
||||
const response = await this.runSkillOnce({
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
});
|
||||
const normalized = this.normalizeControlTokens(response, skill.name);
|
||||
const verified = await this.applyRuntimeVerifier({
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
response: normalized,
|
||||
skill,
|
||||
});
|
||||
const escalated = this.applyEscalationTrigger({
|
||||
budget,
|
||||
request,
|
||||
response: verified,
|
||||
skill,
|
||||
});
|
||||
|
||||
const handoff = readHandoffRequest(escalated);
|
||||
if (!handoff) {
|
||||
return escalated;
|
||||
}
|
||||
|
||||
const next = this.prepareHandoff({
|
||||
budget,
|
||||
chain,
|
||||
currentSkill: skill.name,
|
||||
depth,
|
||||
handoff,
|
||||
history,
|
||||
request,
|
||||
response: escalated,
|
||||
});
|
||||
|
||||
if (!next) {
|
||||
return this.makeSupervisorFallback(request, handoff.reason);
|
||||
}
|
||||
|
||||
return this.dispatch(next);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async runSkillOnce(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
return withSkillSpan(
|
||||
input.skill.name,
|
||||
{
|
||||
"agent.handoff_chain": input.chain.join(">"),
|
||||
"agent.handoff_depth": input.depth,
|
||||
"agent.skill": input.skill.name,
|
||||
"agent.trigger": input.request.trigger,
|
||||
},
|
||||
() => input.skill.handle(input.request, input.history),
|
||||
);
|
||||
}
|
||||
|
||||
private normalizeControlTokens(
|
||||
response: AgentResponse,
|
||||
currentSkill: string,
|
||||
): AgentResponse {
|
||||
const parsed = parseControlTokens(response.text);
|
||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||
const metadata = { ...(response.metadata ?? {}) };
|
||||
if (handoff?.kind === "handoff") {
|
||||
metadata.handoffRequest = {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: handoff.reason,
|
||||
risk: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: `Continue handling from ${currentSkill}`,
|
||||
} satisfies HandoffRequest;
|
||||
}
|
||||
return {
|
||||
...response,
|
||||
metadata,
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
private async applyRuntimeVerifier(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
if (!getAgentRuntimeVerifierEnabled()) {
|
||||
return input.response;
|
||||
}
|
||||
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
|
||||
if (!verifier) {
|
||||
return input.response;
|
||||
}
|
||||
const verdict = await verifier({
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (verdict === "pass") {
|
||||
return input.response;
|
||||
}
|
||||
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
|
||||
if (verdict === "retry" && input.depth === 0) {
|
||||
const retryResponse = await this.runSkillOnce({
|
||||
...input,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
runtimeVerifierRetry: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
const normalizedRetry = this.normalizeControlTokens(
|
||||
retryResponse,
|
||||
input.skill.name,
|
||||
);
|
||||
if (readHandoffRequest(normalizedRetry)) {
|
||||
return normalizedRetry;
|
||||
}
|
||||
const retryVerdict = await verifier({
|
||||
request: input.request,
|
||||
response: normalizedRetry,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (retryVerdict === "pass") {
|
||||
return {
|
||||
...normalizedRetry,
|
||||
metadata: {
|
||||
...(normalizedRetry.metadata ?? {}),
|
||||
runtimeVerifier: "retry-pass",
|
||||
},
|
||||
};
|
||||
}
|
||||
metrics.validatorFailTotal.inc({
|
||||
skill: input.skill.name,
|
||||
stage: `retry-${retryVerdict}`,
|
||||
});
|
||||
return this.withRuntimeVerifierHandoff(normalizedRetry);
|
||||
}
|
||||
return this.withRuntimeVerifierHandoff(input.response);
|
||||
}
|
||||
|
||||
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "validation-failed",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "Runtime verifier rejected the response.",
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private applyEscalationTrigger(input: {
|
||||
budget: BudgetTracker;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
}): AgentResponse {
|
||||
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
const decision = decideEscalation({
|
||||
budget: input.budget.snapshot(),
|
||||
capabilityCard: input.skill.capabilityCard?.(),
|
||||
classified: { confidence: input.response.confidence },
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
||||
});
|
||||
if (!decision.handoff) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
return {
|
||||
...input.response,
|
||||
metadata: {
|
||||
...(input.response.metadata ?? {}),
|
||||
handoffRequest: decision.handoff,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private prepareHandoff(input: {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
currentSkill: string;
|
||||
depth: number;
|
||||
handoff: HandoffRequest;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}):
|
||||
| {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}
|
||||
| null {
|
||||
const target = this.resolveHandoffTarget(input);
|
||||
const nextSkill = selectSkillForRequest(this.skills, {
|
||||
...input.request,
|
||||
skill: target.targetAgent,
|
||||
trigger: "handoff",
|
||||
});
|
||||
if (!nextSkill) {
|
||||
return null;
|
||||
}
|
||||
|
||||
input.budget.recordHandoff(nextSkill.name);
|
||||
metrics.handoffTotal.inc({
|
||||
from: input.currentSkill,
|
||||
reason: target.reason,
|
||||
to: nextSkill.name,
|
||||
});
|
||||
|
||||
const nextChain = [...input.chain, nextSkill.name];
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
...input.handoff,
|
||||
reason: target.reason,
|
||||
targetAgent: nextSkill.name,
|
||||
},
|
||||
history: input.history,
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
sourceChain: nextChain,
|
||||
state: {
|
||||
source: input.currentSkill,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
budget: input.budget,
|
||||
chain: nextChain,
|
||||
depth: input.depth + 1,
|
||||
history: input.history,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
handoffPacket: packet,
|
||||
},
|
||||
skill: nextSkill.name,
|
||||
trigger: "handoff",
|
||||
},
|
||||
skill: nextSkill,
|
||||
};
|
||||
}
|
||||
|
||||
private resolveHandoffTarget(input: {
|
||||
budget: BudgetTracker;
|
||||
currentSkill: string;
|
||||
handoff: HandoffRequest;
|
||||
}): { reason: HandoffReason; targetAgent: string } {
|
||||
const requestedTarget = input.handoff.targetAgent;
|
||||
const allowed = this.transitionMatrix?.isAllowed(
|
||||
input.currentSkill,
|
||||
requestedTarget,
|
||||
) ?? true;
|
||||
if (!allowed) {
|
||||
return {
|
||||
reason: "policy-risk",
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
||||
if (verdict === "ok") {
|
||||
return {
|
||||
reason: input.handoff.reason,
|
||||
targetAgent: requestedTarget,
|
||||
};
|
||||
}
|
||||
|
||||
if (verdict === "same-agent-loop") {
|
||||
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
|
||||
}
|
||||
|
||||
return {
|
||||
reason: budgetVerdictToReason(verdict),
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
private makeSupervisorFallback(
|
||||
request: AgentRequest,
|
||||
reason: HandoffReason,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: 0.4,
|
||||
metadata: {
|
||||
supervisorFallbackReason: reason,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||
if (verdict === "same-agent-loop") {
|
||||
return "same-agent-revisit";
|
||||
}
|
||||
if (verdict === "max-hops") {
|
||||
return "max-hop";
|
||||
}
|
||||
return "policy-risk";
|
||||
}
|
||||
|
||||
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
||||
const value = response.metadata?.["handoffRequest"];
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const handoff = value as Partial<HandoffRequest>;
|
||||
if (
|
||||
typeof handoff.targetAgent !== "string" ||
|
||||
typeof handoff.reason !== "string" ||
|
||||
typeof handoff.taskSpec !== "string" ||
|
||||
typeof handoff.confidence !== "number" ||
|
||||
typeof handoff.expectedOutput !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
confidence: handoff.confidence,
|
||||
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
||||
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||
: [],
|
||||
expectedOutput: handoff.expectedOutput,
|
||||
reason: handoff.reason,
|
||||
risk:
|
||||
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
||||
? handoff.risk
|
||||
: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: handoff.taskSpec,
|
||||
...(handoff.userVisibleMessage
|
||||
? { userVisibleMessage: handoff.userVisibleMessage }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user