Unify dispatch and add supervisor routing signals
This commit is contained in:
1 parent
f0b00e5fd2
commit
b63985694a
19 files changed
+1345
-812
No files matched your search
+37
-438
@@ -3,22 +3,14 @@ import type { ConversationStore } from "@conv/conversation-store";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import { getRequestLogger, runWithContext } from "@obs/context";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import { withSkillSpan, withSpan } from "@obs/spans";
|
||||
import { withSpan } from "@obs/spans";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||
import { parseControlTokens } from "@skills/control-token-parser";
|
||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
|
||||
import { buildHandoffPacket } from "./handoff-packet.js";
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
HandoffRequest,
|
||||
} from "./types.js";
|
||||
import { BudgetTracker } from "./budget.js";
|
||||
import { dispatch } from "./dispatch.js";
|
||||
import type { AgentRequest, AgentResponse } from "./types.js";
|
||||
import {
|
||||
getAgentRuntimeVerifierEnabled,
|
||||
getAgentRequestTimeoutMs,
|
||||
@@ -167,14 +159,39 @@ export class Agent {
|
||||
const startedAt = Date.now();
|
||||
const budget = new BudgetTracker();
|
||||
budget.recordVisit(skill.name);
|
||||
const response = await this.dispatch({
|
||||
budget,
|
||||
chain: [skill.name],
|
||||
depth: 0,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
});
|
||||
const response = await dispatch(
|
||||
{
|
||||
budget,
|
||||
chain: [skill.name],
|
||||
depth: 0,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
},
|
||||
{
|
||||
getSkillJudgeConfig,
|
||||
onHandoff: (event) => {
|
||||
metrics.handoffTotal.inc({
|
||||
from: event.from,
|
||||
reason: event.reason,
|
||||
to: event.to,
|
||||
});
|
||||
},
|
||||
onHandoffLoop: (event) => {
|
||||
metrics.handoffLoopTotal.inc({ from: event.from });
|
||||
},
|
||||
onValidatorFail: (event) => {
|
||||
metrics.validatorFailTotal.inc({
|
||||
skill: event.skill,
|
||||
stage: event.stage,
|
||||
});
|
||||
},
|
||||
selectSkillForRequest,
|
||||
shouldApplyRuntimeVerifier: getAgentRuntimeVerifierEnabled,
|
||||
skills: this.skills,
|
||||
transitionMatrix: this.transitionMatrix,
|
||||
},
|
||||
);
|
||||
|
||||
if (response.text.trim()) {
|
||||
await this.conversationStore.appendMessage(
|
||||
@@ -230,422 +247,4 @@ export class Agent {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async dispatch(input: {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
const { budget, chain, depth, history, request, skill } = input;
|
||||
const contextBase = {
|
||||
chatroomId: request.chatroomId,
|
||||
deadlineAt: new Date(
|
||||
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
||||
).toISOString(),
|
||||
requestId: request.requestId,
|
||||
skill: skill.name,
|
||||
startedAt: new Date().toISOString(),
|
||||
trigger: request.trigger,
|
||||
userId: request.userId,
|
||||
};
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
...contextBase,
|
||||
budget: budget.snapshot(),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
},
|
||||
async () => {
|
||||
const response = await this.runSkillOnce({
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
});
|
||||
const normalized = this.normalizeControlTokens(response, skill.name);
|
||||
const verified = await this.applyRuntimeVerifier({
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
response: normalized,
|
||||
skill,
|
||||
});
|
||||
const escalated = this.applyEscalationTrigger({
|
||||
budget,
|
||||
request,
|
||||
response: verified,
|
||||
skill,
|
||||
});
|
||||
const annotated = this.withHandoffMetadata(escalated, chain, depth);
|
||||
|
||||
const handoff = readHandoffRequest(annotated);
|
||||
if (!handoff) {
|
||||
return annotated;
|
||||
}
|
||||
|
||||
const next = this.prepareHandoff({
|
||||
budget,
|
||||
chain,
|
||||
currentSkill: skill.name,
|
||||
depth,
|
||||
handoff,
|
||||
history,
|
||||
request,
|
||||
response: annotated,
|
||||
});
|
||||
|
||||
if (!next) {
|
||||
return this.withHandoffMetadata(
|
||||
this.makeSupervisorFallback(request, handoff.reason),
|
||||
[...chain, "supervisor"],
|
||||
depth + 1,
|
||||
);
|
||||
}
|
||||
|
||||
return this.dispatch(next);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private withHandoffMetadata(
|
||||
response: AgentResponse,
|
||||
chain: string[],
|
||||
depth: number,
|
||||
): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async runSkillOnce(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
return withSkillSpan(
|
||||
input.skill.name,
|
||||
{
|
||||
"agent.handoff_chain": input.chain.join(">"),
|
||||
"agent.handoff_depth": input.depth,
|
||||
"agent.skill": input.skill.name,
|
||||
"agent.trigger": input.request.trigger,
|
||||
},
|
||||
() => input.skill.handle(input.request, input.history),
|
||||
);
|
||||
}
|
||||
|
||||
private normalizeControlTokens(
|
||||
response: AgentResponse,
|
||||
currentSkill: string,
|
||||
): AgentResponse {
|
||||
const parsed = parseControlTokens(response.text);
|
||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||
const metadata = { ...(response.metadata ?? {}) };
|
||||
if (handoff?.kind === "handoff") {
|
||||
metadata.handoffRequest = {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: handoff.reason,
|
||||
risk: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: `Continue handling from ${currentSkill}`,
|
||||
} satisfies HandoffRequest;
|
||||
}
|
||||
return {
|
||||
...response,
|
||||
metadata,
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
private async applyRuntimeVerifier(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
if (!getAgentRuntimeVerifierEnabled()) {
|
||||
return input.response;
|
||||
}
|
||||
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
|
||||
if (!verifier) {
|
||||
return input.response;
|
||||
}
|
||||
const verdict = await verifier({
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (verdict === "pass") {
|
||||
return input.response;
|
||||
}
|
||||
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
|
||||
if (verdict === "retry" && input.depth === 0) {
|
||||
const retryResponse = await this.runSkillOnce({
|
||||
...input,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
runtimeVerifierRetry: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
const normalizedRetry = this.normalizeControlTokens(
|
||||
retryResponse,
|
||||
input.skill.name,
|
||||
);
|
||||
if (readHandoffRequest(normalizedRetry)) {
|
||||
return normalizedRetry;
|
||||
}
|
||||
const retryVerdict = await verifier({
|
||||
request: input.request,
|
||||
response: normalizedRetry,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (retryVerdict === "pass") {
|
||||
return {
|
||||
...normalizedRetry,
|
||||
metadata: {
|
||||
...(normalizedRetry.metadata ?? {}),
|
||||
runtimeVerifier: "retry-pass",
|
||||
},
|
||||
};
|
||||
}
|
||||
metrics.validatorFailTotal.inc({
|
||||
skill: input.skill.name,
|
||||
stage: `retry-${retryVerdict}`,
|
||||
});
|
||||
return this.withRuntimeVerifierHandoff(normalizedRetry);
|
||||
}
|
||||
return this.withRuntimeVerifierHandoff(input.response);
|
||||
}
|
||||
|
||||
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "validation-failed",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "Runtime verifier rejected the response.",
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private applyEscalationTrigger(input: {
|
||||
budget: BudgetTracker;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
}): AgentResponse {
|
||||
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
const decision = decideEscalation({
|
||||
budget: input.budget.snapshot(),
|
||||
capabilityCard: input.skill.capabilityCard?.(),
|
||||
classified: { confidence: input.response.confidence },
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
||||
});
|
||||
if (!decision.handoff) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
return {
|
||||
...input.response,
|
||||
metadata: {
|
||||
...(input.response.metadata ?? {}),
|
||||
handoffRequest: decision.handoff,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private prepareHandoff(input: {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
currentSkill: string;
|
||||
depth: number;
|
||||
handoff: HandoffRequest;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}):
|
||||
| {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}
|
||||
| null {
|
||||
const target = this.resolveHandoffTarget(input);
|
||||
const nextSkill = selectSkillForRequest(this.skills, {
|
||||
...input.request,
|
||||
skill: target.targetAgent,
|
||||
trigger: "handoff",
|
||||
});
|
||||
if (!nextSkill) {
|
||||
return null;
|
||||
}
|
||||
|
||||
input.budget.recordHandoff(nextSkill.name);
|
||||
metrics.handoffTotal.inc({
|
||||
from: input.currentSkill,
|
||||
reason: target.reason,
|
||||
to: nextSkill.name,
|
||||
});
|
||||
|
||||
const nextChain = [...input.chain, nextSkill.name];
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
...input.handoff,
|
||||
reason: target.reason,
|
||||
targetAgent: nextSkill.name,
|
||||
},
|
||||
history: input.history,
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
sourceChain: nextChain,
|
||||
state: {
|
||||
source: input.currentSkill,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
budget: input.budget,
|
||||
chain: nextChain,
|
||||
depth: input.depth + 1,
|
||||
history: input.history,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
handoffPacket: packet,
|
||||
},
|
||||
skill: nextSkill.name,
|
||||
trigger: "handoff",
|
||||
},
|
||||
skill: nextSkill,
|
||||
};
|
||||
}
|
||||
|
||||
private resolveHandoffTarget(input: {
|
||||
budget: BudgetTracker;
|
||||
currentSkill: string;
|
||||
handoff: HandoffRequest;
|
||||
}): { reason: HandoffReason; targetAgent: string } {
|
||||
const requestedTarget = input.handoff.targetAgent;
|
||||
const allowed = this.transitionMatrix?.isAllowed(
|
||||
input.currentSkill,
|
||||
requestedTarget,
|
||||
) ?? true;
|
||||
if (!allowed) {
|
||||
return {
|
||||
reason: "policy-risk",
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
||||
if (verdict === "ok") {
|
||||
return {
|
||||
reason: input.handoff.reason,
|
||||
targetAgent: requestedTarget,
|
||||
};
|
||||
}
|
||||
|
||||
if (verdict === "same-agent-loop") {
|
||||
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
|
||||
}
|
||||
|
||||
return {
|
||||
reason: budgetVerdictToReason(verdict),
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
private makeSupervisorFallback(
|
||||
request: AgentRequest,
|
||||
reason: HandoffReason,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: 0.4,
|
||||
metadata: {
|
||||
supervisorFallbackReason: reason,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||
if (verdict === "same-agent-loop") {
|
||||
return "same-agent-revisit";
|
||||
}
|
||||
if (verdict === "max-hops") {
|
||||
return "max-hop";
|
||||
}
|
||||
return "policy-risk";
|
||||
}
|
||||
|
||||
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
||||
const value = response.metadata?.["handoffRequest"];
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const handoff = value as Partial<HandoffRequest>;
|
||||
if (
|
||||
typeof handoff.targetAgent !== "string" ||
|
||||
typeof handoff.reason !== "string" ||
|
||||
typeof handoff.taskSpec !== "string" ||
|
||||
typeof handoff.confidence !== "number" ||
|
||||
typeof handoff.expectedOutput !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
confidence: handoff.confidence,
|
||||
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
||||
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||
: [],
|
||||
expectedOutput: handoff.expectedOutput,
|
||||
reason: handoff.reason,
|
||||
risk:
|
||||
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
||||
? handoff.risk
|
||||
: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: handoff.taskSpec,
|
||||
...(handoff.userVisibleMessage
|
||||
? { userVisibleMessage: handoff.userVisibleMessage }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,581 @@
|
||||
import { getSkillJudgeConfig as defaultGetSkillJudgeConfig } from "@eval/judges/skill-config";
|
||||
import type { SkillJudgeConfig } from "@eval/judges/skill-config";
|
||||
import { runWithContext } from "@obs/context";
|
||||
import { withSkillSpan } from "@obs/spans";
|
||||
import { parseControlTokens } from "@skills/control-token-parser";
|
||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||
import { getAgentRequestTimeoutMs, getSafetyMarginMs } from "../config.js";
|
||||
import type { BudgetTracker, BudgetVerdict } from "./budget.js";
|
||||
import { buildHandoffPacket } from "./handoff-packet.js";
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
HandoffRequest,
|
||||
} from "./types.js";
|
||||
|
||||
export interface HandoffEvent {
|
||||
from: string;
|
||||
reason: HandoffReason;
|
||||
to: string;
|
||||
}
|
||||
|
||||
export interface ValidatorFailEvent {
|
||||
skill: string;
|
||||
stage: string;
|
||||
}
|
||||
|
||||
export interface EvalDispatchPlanStep {
|
||||
confidence?: number | undefined;
|
||||
expectedOutput?: string | undefined;
|
||||
from: string;
|
||||
reason: HandoffReason;
|
||||
risk?: HandoffRequest["risk"] | undefined;
|
||||
taskSpec?: string | undefined;
|
||||
to: string;
|
||||
}
|
||||
|
||||
export interface EvalDispatchPlan {
|
||||
maxDepth?: number | undefined;
|
||||
steps: EvalDispatchPlanStep[];
|
||||
useActualSkills?: boolean | undefined;
|
||||
}
|
||||
|
||||
export interface DispatchDependencies {
|
||||
evalPlan?: EvalDispatchPlan | undefined;
|
||||
getSkillJudgeConfig?: ((skillName: string) => SkillJudgeConfig) | undefined;
|
||||
onHandoff?: ((event: HandoffEvent) => void) | undefined;
|
||||
onHandoffLoop?: ((event: { from: string }) => void) | undefined;
|
||||
onValidatorFail?: ((event: ValidatorFailEvent) => void) | undefined;
|
||||
selectSkillForRequest: (
|
||||
skills: readonly Skill[],
|
||||
request: AgentRequest,
|
||||
) => Skill | null;
|
||||
shouldApplyRuntimeVerifier: () => boolean;
|
||||
skills: readonly Skill[];
|
||||
transitionMatrix?: TransitionMatrix | undefined;
|
||||
}
|
||||
|
||||
export interface DispatchInput {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
depth: number;
|
||||
evalPlanIndex?: number | undefined;
|
||||
history: ConversationMessage[];
|
||||
lastHandoff?: HandoffRequest | undefined;
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}
|
||||
|
||||
export async function dispatch(
|
||||
input: DispatchInput,
|
||||
deps: DispatchDependencies,
|
||||
): Promise<AgentResponse> {
|
||||
const { budget, chain, depth, history, lastHandoff, request, skill } = input;
|
||||
const contextBase = {
|
||||
chatroomId: request.chatroomId,
|
||||
deadlineAt: new Date(
|
||||
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
||||
).toISOString(),
|
||||
requestId: request.requestId,
|
||||
skill: skill.name,
|
||||
startedAt: new Date().toISOString(),
|
||||
trigger: request.trigger,
|
||||
userId: request.userId,
|
||||
};
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
...contextBase,
|
||||
budget: budget.snapshot(),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
},
|
||||
async () => {
|
||||
const planStep = getCurrentPlanStep(input, deps);
|
||||
const response = planStep
|
||||
? plannedHandoffResponse(request, planStep)
|
||||
: deps.evalPlan && deps.evalPlan.useActualSkills !== true
|
||||
? templateEvalResponse(request, skill.name)
|
||||
: await runSkillOnce({
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
skill,
|
||||
});
|
||||
const normalized = normalizeControlTokens(response, skill.name);
|
||||
const verified = await applyRuntimeVerifier(
|
||||
{
|
||||
chain,
|
||||
depth,
|
||||
history,
|
||||
request,
|
||||
response: normalized,
|
||||
skill,
|
||||
},
|
||||
deps,
|
||||
);
|
||||
const escalated = applyEscalationTrigger({
|
||||
budget,
|
||||
request,
|
||||
response: verified,
|
||||
skill,
|
||||
});
|
||||
const annotated = withHandoffMetadata(
|
||||
escalated,
|
||||
chain,
|
||||
depth,
|
||||
lastHandoff,
|
||||
);
|
||||
|
||||
const handoff = readHandoffRequest(annotated);
|
||||
if (!handoff) {
|
||||
return annotated;
|
||||
}
|
||||
|
||||
const next = prepareHandoff(
|
||||
{
|
||||
budget,
|
||||
chain,
|
||||
currentSkill: skill.name,
|
||||
depth,
|
||||
evalPlanIndex: input.evalPlanIndex ?? 0,
|
||||
handoff,
|
||||
history,
|
||||
planStepUsed: Boolean(planStep),
|
||||
request,
|
||||
response: annotated,
|
||||
},
|
||||
deps,
|
||||
);
|
||||
|
||||
if (!next) {
|
||||
const fallbackHandoff: HandoffRequest = {
|
||||
...handoff,
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
return withHandoffMetadata(
|
||||
makeSupervisorFallback(request, handoff.reason),
|
||||
[...chain, "supervisor"],
|
||||
depth + 1,
|
||||
fallbackHandoff,
|
||||
);
|
||||
}
|
||||
|
||||
const maxDepth = deps.evalPlan?.maxDepth ?? 8;
|
||||
if (next.depth > maxDepth) {
|
||||
const fallbackHandoff: HandoffRequest = {
|
||||
confidence: 0.4,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "max-hop",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "Dispatch exceeded max loop depth.",
|
||||
};
|
||||
const fallbackChain =
|
||||
chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"];
|
||||
return withHandoffMetadata(
|
||||
makeSupervisorFallback(request, "max-hop"),
|
||||
fallbackChain,
|
||||
fallbackChain.length - 1,
|
||||
fallbackHandoff,
|
||||
);
|
||||
}
|
||||
|
||||
return dispatch(next, deps);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function withHandoffMetadata(
|
||||
response: AgentResponse,
|
||||
chain: string[],
|
||||
depth: number,
|
||||
lastHandoff?: HandoffRequest,
|
||||
): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeControlTokens(
|
||||
response: AgentResponse,
|
||||
currentSkill: string,
|
||||
): AgentResponse {
|
||||
const parsed = parseControlTokens(response.text);
|
||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||
const metadata = { ...(response.metadata ?? {}) };
|
||||
if (handoff?.kind === "handoff") {
|
||||
metadata.handoffRequest = {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: handoff.reason,
|
||||
risk: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: `Continue handling from ${currentSkill}`,
|
||||
} satisfies HandoffRequest;
|
||||
}
|
||||
return {
|
||||
...response,
|
||||
metadata,
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
export function readHandoffRequest(
|
||||
response: AgentResponse,
|
||||
): HandoffRequest | null {
|
||||
const value = response.metadata?.["handoffRequest"];
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const handoff = value as Partial<HandoffRequest>;
|
||||
if (
|
||||
typeof handoff.targetAgent !== "string" ||
|
||||
typeof handoff.reason !== "string" ||
|
||||
typeof handoff.taskSpec !== "string" ||
|
||||
typeof handoff.confidence !== "number" ||
|
||||
typeof handoff.expectedOutput !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
confidence: handoff.confidence,
|
||||
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
||||
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||
: [],
|
||||
expectedOutput: handoff.expectedOutput,
|
||||
reason: handoff.reason,
|
||||
risk:
|
||||
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
||||
? handoff.risk
|
||||
: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: handoff.taskSpec,
|
||||
...(handoff.userVisibleMessage
|
||||
? { userVisibleMessage: handoff.userVisibleMessage }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||
if (verdict === "same-agent-loop") {
|
||||
return "same-agent-revisit";
|
||||
}
|
||||
if (verdict === "max-hops") {
|
||||
return "max-hop";
|
||||
}
|
||||
return "policy-risk";
|
||||
}
|
||||
|
||||
async function runSkillOnce(input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
skill: Skill;
|
||||
}): Promise<AgentResponse> {
|
||||
return withSkillSpan(
|
||||
input.skill.name,
|
||||
{
|
||||
"agent.handoff_chain": input.chain.join(">"),
|
||||
"agent.handoff_depth": input.depth,
|
||||
"agent.skill": input.skill.name,
|
||||
"agent.trigger": input.request.trigger,
|
||||
},
|
||||
() => input.skill.handle(input.request, input.history),
|
||||
);
|
||||
}
|
||||
|
||||
async function applyRuntimeVerifier(
|
||||
input: {
|
||||
chain: string[];
|
||||
depth: number;
|
||||
history: ConversationMessage[];
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
},
|
||||
deps: DispatchDependencies,
|
||||
): Promise<AgentResponse> {
|
||||
if (!deps.shouldApplyRuntimeVerifier()) {
|
||||
return input.response;
|
||||
}
|
||||
const judgeConfig = deps.getSkillJudgeConfig ?? defaultGetSkillJudgeConfig;
|
||||
const verifier = judgeConfig(input.skill.name).runtimeVerifier;
|
||||
if (!verifier) {
|
||||
return input.response;
|
||||
}
|
||||
const verdict = await verifier({
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (verdict === "pass") {
|
||||
return input.response;
|
||||
}
|
||||
deps.onValidatorFail?.({ skill: input.skill.name, stage: verdict });
|
||||
if (verdict === "retry" && input.depth === 0) {
|
||||
const retryResponse = await runSkillOnce({
|
||||
...input,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
runtimeVerifierRetry: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
const normalizedRetry = normalizeControlTokens(
|
||||
retryResponse,
|
||||
input.skill.name,
|
||||
);
|
||||
if (readHandoffRequest(normalizedRetry)) {
|
||||
return normalizedRetry;
|
||||
}
|
||||
const retryVerdict = await verifier({
|
||||
request: input.request,
|
||||
response: normalizedRetry,
|
||||
skill: input.skill.name,
|
||||
});
|
||||
if (retryVerdict === "pass") {
|
||||
return {
|
||||
...normalizedRetry,
|
||||
metadata: {
|
||||
...(normalizedRetry.metadata ?? {}),
|
||||
runtimeVerifier: "retry-pass",
|
||||
},
|
||||
};
|
||||
}
|
||||
deps.onValidatorFail?.({
|
||||
skill: input.skill.name,
|
||||
stage: `retry-${retryVerdict}`,
|
||||
});
|
||||
return withRuntimeVerifierHandoff(normalizedRetry);
|
||||
}
|
||||
return withRuntimeVerifierHandoff(input.response);
|
||||
}
|
||||
|
||||
function withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "validation-failed",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "Runtime verifier rejected the response.",
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function applyEscalationTrigger(input: {
|
||||
budget: BudgetTracker;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
skill: Skill;
|
||||
}): AgentResponse {
|
||||
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
const decision = decideEscalation({
|
||||
budget: input.budget.snapshot(),
|
||||
capabilityCard: input.skill.capabilityCard?.(),
|
||||
classified: { confidence: input.response.confidence },
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
||||
});
|
||||
if (!decision.handoff) {
|
||||
return input.response;
|
||||
}
|
||||
|
||||
return {
|
||||
...input.response,
|
||||
metadata: {
|
||||
...(input.response.metadata ?? {}),
|
||||
handoffRequest: decision.handoff,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function prepareHandoff(
|
||||
input: {
|
||||
budget: BudgetTracker;
|
||||
chain: string[];
|
||||
currentSkill: string;
|
||||
depth: number;
|
||||
evalPlanIndex: number;
|
||||
handoff: HandoffRequest;
|
||||
history: ConversationMessage[];
|
||||
planStepUsed: boolean;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
},
|
||||
deps: DispatchDependencies,
|
||||
): DispatchInput | null {
|
||||
const target = resolveHandoffTarget(input, deps);
|
||||
const nextSkill = deps.selectSkillForRequest(deps.skills, {
|
||||
...input.request,
|
||||
skill: target.targetAgent,
|
||||
trigger: "handoff",
|
||||
});
|
||||
if (!nextSkill) {
|
||||
return null;
|
||||
}
|
||||
|
||||
input.budget.recordHandoff(nextSkill.name);
|
||||
deps.onHandoff?.({
|
||||
from: input.currentSkill,
|
||||
reason: target.reason,
|
||||
to: nextSkill.name,
|
||||
});
|
||||
|
||||
const nextChain = [...input.chain, nextSkill.name];
|
||||
const lastHandoff: HandoffRequest = {
|
||||
...input.handoff,
|
||||
reason: target.reason,
|
||||
targetAgent: nextSkill.name,
|
||||
};
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: lastHandoff,
|
||||
history: input.history,
|
||||
request: input.request,
|
||||
response: input.response,
|
||||
sourceChain: nextChain,
|
||||
state: {
|
||||
source: input.currentSkill,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
budget: input.budget,
|
||||
chain: nextChain,
|
||||
depth: input.depth + 1,
|
||||
evalPlanIndex: input.evalPlanIndex + (input.planStepUsed ? 1 : 0),
|
||||
history: input.history,
|
||||
lastHandoff,
|
||||
request: {
|
||||
...input.request,
|
||||
context: {
|
||||
...(input.request.context ?? {}),
|
||||
handoffPacket: packet,
|
||||
},
|
||||
skill: nextSkill.name,
|
||||
trigger: "handoff",
|
||||
},
|
||||
skill: nextSkill,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveHandoffTarget(
|
||||
input: {
|
||||
budget: BudgetTracker;
|
||||
currentSkill: string;
|
||||
handoff: HandoffRequest;
|
||||
},
|
||||
deps: DispatchDependencies,
|
||||
): { reason: HandoffReason; targetAgent: string } {
|
||||
const requestedTarget = input.handoff.targetAgent;
|
||||
const allowed =
|
||||
deps.transitionMatrix?.isAllowed(input.currentSkill, requestedTarget) ?? true;
|
||||
if (!allowed) {
|
||||
return {
|
||||
reason: "policy-risk",
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
||||
if (verdict === "ok") {
|
||||
return {
|
||||
reason: input.handoff.reason,
|
||||
targetAgent: requestedTarget,
|
||||
};
|
||||
}
|
||||
|
||||
if (verdict === "same-agent-loop") {
|
||||
deps.onHandoffLoop?.({ from: input.currentSkill });
|
||||
}
|
||||
|
||||
return {
|
||||
reason: budgetVerdictToReason(verdict),
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
}
|
||||
|
||||
function makeSupervisorFallback(
|
||||
request: AgentRequest,
|
||||
reason: HandoffReason,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: 0.4,
|
||||
metadata: {
|
||||
supervisorFallbackReason: reason,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
|
||||
function getCurrentPlanStep(
|
||||
input: DispatchInput,
|
||||
deps: DispatchDependencies,
|
||||
): EvalDispatchPlanStep | undefined {
|
||||
const index = input.evalPlanIndex ?? 0;
|
||||
const step = deps.evalPlan?.steps[index];
|
||||
return step?.from === input.skill.name ? step : undefined;
|
||||
}
|
||||
|
||||
function plannedHandoffResponse(
|
||||
request: AgentRequest,
|
||||
step: EvalDispatchPlanStep,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: step.confidence ?? 0.72,
|
||||
metadata: {
|
||||
handoffRequest: {
|
||||
confidence: step.confidence ?? 0.72,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: step.expectedOutput ?? "text",
|
||||
reason: step.reason,
|
||||
risk: step.risk ?? "medium",
|
||||
targetAgent: step.to,
|
||||
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: `handoff ${step.from} to ${step.to}`,
|
||||
};
|
||||
}
|
||||
|
||||
function templateEvalResponse(
|
||||
request: AgentRequest,
|
||||
skillName: string,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: 1,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: `handoff eval final: ${skillName}`,
|
||||
};
|
||||
}
|
||||
@@ -229,6 +229,18 @@ export function getSupervisorReviewResolutionTtlSec(): number {
|
||||
);
|
||||
}
|
||||
|
||||
export function getSupervisorResolutionsStreamMaxLen(): number {
|
||||
return getPositiveNumber("SUPERVISOR_RESOLUTIONS_STREAM_MAXLEN", 100_000);
|
||||
}
|
||||
|
||||
export function getSupervisorLlmEnabled(): boolean {
|
||||
return getBoolean("AGENT_SUPERVISOR_LLM_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getSupervisorLlmConfidenceThreshold(): number {
|
||||
return getRatioNumber("AGENT_SUPERVISOR_LLM_CONFIDENCE_THRESHOLD", 0.6);
|
||||
}
|
||||
|
||||
export function getEvalCalibrationMinCorr(): number {
|
||||
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
||||
}
|
||||
|
||||
+68
-337
@@ -1,16 +1,10 @@
|
||||
import { BudgetTracker, type BudgetVerdict } from "@agent/budget";
|
||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||
import { selectSkillForRequest } from "@agent/agent";
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
HandoffReason,
|
||||
HandoffRequest,
|
||||
} from "@agent/types";
|
||||
import { parseControlTokens } from "@skills/control-token-parser";
|
||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||
import { BudgetTracker } from "@agent/budget";
|
||||
import { dispatch, type EvalDispatchPlan, type EvalDispatchPlanStep } from "@agent/dispatch";
|
||||
import type { AgentRequest, AgentResponse, ConversationMessage, HandoffReason } from "@agent/types";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||
|
||||
export interface EvalDispatchResult {
|
||||
chain: string[];
|
||||
@@ -18,29 +12,43 @@ export interface EvalDispatchResult {
|
||||
response: AgentResponse;
|
||||
}
|
||||
|
||||
interface EvalHandoffPlanStep {
|
||||
confidence?: number | undefined;
|
||||
expectedOutput?: string | undefined;
|
||||
from: string;
|
||||
reason: HandoffReason;
|
||||
risk?: HandoffRequest["risk"] | undefined;
|
||||
taskSpec?: string | undefined;
|
||||
to: string;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
|
||||
const HANDOFF_REASONS = [
|
||||
"out-of-scope",
|
||||
"missing-tool",
|
||||
"low-confidence",
|
||||
"validation-failed",
|
||||
"same-agent-revisit",
|
||||
"max-hop",
|
||||
"high-risk-action",
|
||||
"policy-risk",
|
||||
] as const;
|
||||
|
||||
function isHandoffReason(value: unknown): value is HandoffReason {
|
||||
return HANDOFF_REASONS.includes(value as HandoffReason);
|
||||
}
|
||||
|
||||
function readPositiveNumber(
|
||||
context: Record<string, unknown> | undefined,
|
||||
key: string,
|
||||
fallback: number,
|
||||
): number {
|
||||
const value = context?.[key];
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function readPlan(request: AgentRequest): EvalDispatchPlanStep[] {
|
||||
const rawPlan = asRecord(request.context)["evalHandoffPlan"];
|
||||
if (!Array.isArray(rawPlan)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return rawPlan.flatMap((item): EvalHandoffPlanStep[] => {
|
||||
return rawPlan.flatMap((item): EvalDispatchPlanStep[] => {
|
||||
const record = asRecord(item);
|
||||
const from = record["from"];
|
||||
const to = record["to"];
|
||||
@@ -55,14 +63,8 @@ function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
|
||||
|
||||
return [
|
||||
{
|
||||
confidence:
|
||||
typeof record["confidence"] === "number"
|
||||
? record["confidence"]
|
||||
: undefined,
|
||||
expectedOutput:
|
||||
typeof record["expectedOutput"] === "string"
|
||||
? record["expectedOutput"]
|
||||
: undefined,
|
||||
confidence: typeof record["confidence"] === "number" ? record["confidence"] : undefined,
|
||||
expectedOutput: typeof record["expectedOutput"] === "string" ? record["expectedOutput"] : undefined,
|
||||
from,
|
||||
reason,
|
||||
risk:
|
||||
@@ -71,198 +73,26 @@ function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
|
||||
record["risk"] === "high"
|
||||
? record["risk"]
|
||||
: undefined,
|
||||
taskSpec:
|
||||
typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
|
||||
taskSpec: typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
|
||||
to,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
function isHandoffReason(value: unknown): value is HandoffReason {
|
||||
return (
|
||||
value === "out-of-scope" ||
|
||||
value === "missing-tool" ||
|
||||
value === "low-confidence" ||
|
||||
value === "validation-failed" ||
|
||||
value === "same-agent-revisit" ||
|
||||
value === "max-hop" ||
|
||||
value === "high-risk-action" ||
|
||||
value === "policy-risk"
|
||||
);
|
||||
}
|
||||
|
||||
function readPositiveNumber(
|
||||
context: Record<string, unknown> | undefined,
|
||||
key: string,
|
||||
fallback: number,
|
||||
): number {
|
||||
const value = context?.[key];
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0
|
||||
? value
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||
if (verdict === "same-agent-loop") {
|
||||
return "same-agent-revisit";
|
||||
}
|
||||
if (verdict === "max-hops") {
|
||||
return "max-hop";
|
||||
}
|
||||
return "policy-risk";
|
||||
}
|
||||
|
||||
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
||||
const value = response.metadata?.["handoffRequest"];
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return null;
|
||||
}
|
||||
const request = value as Partial<HandoffRequest>;
|
||||
if (
|
||||
typeof request.targetAgent !== "string" ||
|
||||
typeof request.reason !== "string" ||
|
||||
typeof request.taskSpec !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
confidence:
|
||||
typeof request.confidence === "number" ? request.confidence : response.confidence,
|
||||
evidenceRefs: Array.isArray(request.evidenceRefs)
|
||||
? request.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||
: [],
|
||||
expectedOutput:
|
||||
typeof request.expectedOutput === "string"
|
||||
? request.expectedOutput
|
||||
: "text",
|
||||
reason: request.reason,
|
||||
risk:
|
||||
request.risk === "low" || request.risk === "medium" || request.risk === "high"
|
||||
? request.risk
|
||||
: "medium",
|
||||
targetAgent: request.targetAgent,
|
||||
taskSpec: request.taskSpec,
|
||||
...(request.userVisibleMessage
|
||||
? { userVisibleMessage: request.userVisibleMessage }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeControlTokens(
|
||||
response: AgentResponse,
|
||||
currentSkill: string,
|
||||
): AgentResponse {
|
||||
const parsed = parseControlTokens(response.text);
|
||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||
if (handoff?.kind !== "handoff") {
|
||||
return {
|
||||
...response,
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: {
|
||||
confidence: response.confidence,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: handoff.reason,
|
||||
risk: "medium",
|
||||
targetAgent: handoff.targetAgent,
|
||||
taskSpec: `Continue handling from ${currentSkill}`,
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
text: parsed.stripped,
|
||||
};
|
||||
}
|
||||
|
||||
function withHandoffMetadata(
|
||||
response: AgentResponse,
|
||||
chain: string[],
|
||||
depth: number,
|
||||
lastHandoff: HandoffRequest | undefined,
|
||||
): AgentResponse {
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function plannedHandoffResponse(
|
||||
request: AgentRequest,
|
||||
step: EvalHandoffPlanStep,
|
||||
): AgentResponse {
|
||||
return {
|
||||
confidence: step.confidence ?? 0.72,
|
||||
metadata: {
|
||||
handoffRequest: {
|
||||
confidence: step.confidence ?? 0.72,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: step.expectedOutput ?? "text",
|
||||
reason: step.reason,
|
||||
risk: step.risk ?? "medium",
|
||||
targetAgent: step.to,
|
||||
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
|
||||
} satisfies HandoffRequest,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: `handoff ${step.from} to ${step.to}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function runActualSkill(
|
||||
request: AgentRequest,
|
||||
history: ConversationMessage[],
|
||||
skill: Skill,
|
||||
budget: BudgetTracker,
|
||||
): Promise<AgentResponse> {
|
||||
const response = normalizeControlTokens(
|
||||
await skill.handle(request, history),
|
||||
skill.name,
|
||||
);
|
||||
if (!response.text.trim() || readHandoffRequest(response)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const decision = decideEscalation({
|
||||
budget: budget.snapshot(),
|
||||
capabilityCard: skill.capabilityCard?.(),
|
||||
classified: { confidence: response.confidence },
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: `Review ${skill.name} response for ${request.trigger}.`,
|
||||
});
|
||||
if (!decision.handoff) {
|
||||
return response;
|
||||
}
|
||||
|
||||
return {
|
||||
...response,
|
||||
metadata: {
|
||||
...(response.metadata ?? {}),
|
||||
handoffRequest: decision.handoff,
|
||||
},
|
||||
};
|
||||
function asStringArray(value: unknown): string[] {
|
||||
return Array.isArray(value)
|
||||
? value.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
}
|
||||
|
||||
export async function runEvalDispatch(input: {
|
||||
history?: ConversationMessage[] | undefined;
|
||||
request: AgentRequest;
|
||||
skills: readonly Skill[];
|
||||
transitionMatrix?: TransitionMatrix | undefined;
|
||||
}): Promise<EvalDispatchResult> {
|
||||
const context = input.request.context;
|
||||
const plan = readPlan(input.request);
|
||||
const useActualSkills = asRecord(context)["evalUseActualSkills"] === true;
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5),
|
||||
@@ -274,139 +104,40 @@ export async function runEvalDispatch(input: {
|
||||
),
|
||||
maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20),
|
||||
});
|
||||
|
||||
let request = input.request;
|
||||
let skill = selectSkillForRequest(input.skills, request);
|
||||
const skill = selectSkillForRequest(input.skills, input.request);
|
||||
if (!skill) {
|
||||
throw new Error(`No skill can handle eval request: ${request.skill}`);
|
||||
throw new Error(`No skill can handle eval request: ${input.request.skill}`);
|
||||
}
|
||||
|
||||
let chain = [skill.name];
|
||||
let lastHandoff: HandoffRequest | undefined;
|
||||
let planIndex = 0;
|
||||
budget.recordVisit(skill.name);
|
||||
|
||||
for (let depth = 0; depth <= 8; depth++) {
|
||||
const planned = plan[planIndex]?.from === skill.name ? plan[planIndex] : undefined;
|
||||
const response = planned
|
||||
? plannedHandoffResponse(request, planned)
|
||||
: useActualSkills
|
||||
? await runActualSkill(request, input.history ?? [], skill, budget)
|
||||
: {
|
||||
confidence: 1,
|
||||
requestId: request.requestId,
|
||||
source: "template" as const,
|
||||
text: `handoff eval final: ${skill.name}`,
|
||||
};
|
||||
if (planned) {
|
||||
planIndex += 1;
|
||||
}
|
||||
|
||||
const handoff = readHandoffRequest(response);
|
||||
if (!handoff) {
|
||||
const finalResponse = withHandoffMetadata(
|
||||
response,
|
||||
chain,
|
||||
chain.length - 1,
|
||||
lastHandoff,
|
||||
);
|
||||
return {
|
||||
chain,
|
||||
finalSkill: skill.name,
|
||||
response: finalResponse,
|
||||
};
|
||||
}
|
||||
|
||||
const verdict = budget.assertBeforeHandoff(handoff.targetAgent);
|
||||
const resolved: HandoffRequest =
|
||||
verdict === "ok"
|
||||
? handoff
|
||||
: {
|
||||
...handoff,
|
||||
reason: budgetVerdictToReason(verdict),
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
const nextSkill = selectSkillForRequest(input.skills, {
|
||||
...request,
|
||||
skill: resolved.targetAgent,
|
||||
trigger: "handoff",
|
||||
});
|
||||
if (!nextSkill) {
|
||||
const fallbackChain = [...chain, "supervisor"];
|
||||
const fallbackHandoff = {
|
||||
...resolved,
|
||||
targetAgent: "supervisor",
|
||||
};
|
||||
return {
|
||||
chain: fallbackChain,
|
||||
finalSkill: "supervisor",
|
||||
response: withHandoffMetadata(
|
||||
{
|
||||
confidence: 0.4,
|
||||
requestId: request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
},
|
||||
fallbackChain,
|
||||
fallbackChain.length - 1,
|
||||
fallbackHandoff,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
budget.recordHandoff(nextSkill.name);
|
||||
const nextChain = [...chain, nextSkill.name];
|
||||
lastHandoff = {
|
||||
...resolved,
|
||||
targetAgent: nextSkill.name,
|
||||
};
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: lastHandoff,
|
||||
history: input.history,
|
||||
request,
|
||||
response,
|
||||
sourceChain: nextChain,
|
||||
state: {
|
||||
source: skill.name,
|
||||
},
|
||||
});
|
||||
|
||||
request = {
|
||||
...request,
|
||||
context: {
|
||||
...(request.context ?? {}),
|
||||
handoffPacket: packet,
|
||||
},
|
||||
skill: nextSkill.name,
|
||||
trigger: "handoff",
|
||||
};
|
||||
skill = nextSkill;
|
||||
chain = nextChain;
|
||||
}
|
||||
|
||||
const finalHandoff: HandoffRequest = {
|
||||
confidence: 0.4,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "max-hop",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "Eval dispatch exceeded max loop depth.",
|
||||
const plan: EvalDispatchPlan = {
|
||||
maxDepth: readPositiveNumber(context, "evalMaxDispatchDepth", 8),
|
||||
steps: readPlan(input.request),
|
||||
useActualSkills: asRecord(context)["evalUseActualSkills"] === true,
|
||||
};
|
||||
const fallbackChain = chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"];
|
||||
const response = await dispatch(
|
||||
{
|
||||
budget,
|
||||
chain: [skill.name],
|
||||
depth: 0,
|
||||
history: input.history ?? [],
|
||||
request: input.request,
|
||||
skill,
|
||||
},
|
||||
{
|
||||
evalPlan: plan,
|
||||
getSkillJudgeConfig,
|
||||
selectSkillForRequest,
|
||||
shouldApplyRuntimeVerifier: () => false,
|
||||
skills: input.skills,
|
||||
transitionMatrix: input.transitionMatrix,
|
||||
},
|
||||
);
|
||||
const chain = asStringArray(response.metadata?.["handoffChain"]);
|
||||
|
||||
return {
|
||||
chain: fallbackChain,
|
||||
finalSkill: "supervisor",
|
||||
response: withHandoffMetadata(
|
||||
{
|
||||
confidence: 0.4,
|
||||
requestId: input.request.requestId,
|
||||
source: "template",
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
},
|
||||
fallbackChain,
|
||||
fallbackChain.length - 1,
|
||||
finalHandoff,
|
||||
),
|
||||
chain,
|
||||
finalSkill: chain.at(-1) ?? skill.name,
|
||||
response,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import { BACKGROUND_STREAM_KEY } from "./shadow-bus.js";
|
||||
import { loadSupervisorOutcomes } from "./router-training-signals.js";
|
||||
import {
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
} from "@skills/supervisor/review-queue";
|
||||
|
||||
class FakeGarnet {
|
||||
constructor(private readonly streams: Record<string, StreamEntry[]>) {}
|
||||
|
||||
readStreamRange(key: string): Promise<StreamEntry[]> {
|
||||
return Promise.resolve(this.streams[key] ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
describe("loadSupervisorOutcomes", () => {
|
||||
it("joins supervisor review entries, resolution audit, and shadow eval by requestId", async () => {
|
||||
const garnet = new FakeGarnet({
|
||||
[BACKGROUND_STREAM_KEY]: [
|
||||
{
|
||||
id: "10-0",
|
||||
message: {
|
||||
classified: "{}",
|
||||
createdAt: "2026-05-18T00:00:30.000Z",
|
||||
derived: "{}",
|
||||
input: JSON.stringify({
|
||||
request: {
|
||||
chatroomId: "room-1",
|
||||
nickname: "tester",
|
||||
requestId: "request-1",
|
||||
skill: "onboarding",
|
||||
trigger: "user-message",
|
||||
userId: "user-1",
|
||||
},
|
||||
}),
|
||||
output: JSON.stringify({
|
||||
confidence: 1,
|
||||
requestId: "request-1",
|
||||
source: "template",
|
||||
text: "ok",
|
||||
}),
|
||||
skill: "onboarding",
|
||||
},
|
||||
},
|
||||
],
|
||||
[SUPERVISOR_RESOLUTIONS_STREAM_KEY]: [
|
||||
{
|
||||
id: "2-0",
|
||||
message: {
|
||||
action: "manual-reply",
|
||||
resolvedAt: "2026-05-18T00:02:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
reviewId: "1-0",
|
||||
type: "resolution",
|
||||
},
|
||||
},
|
||||
],
|
||||
[SUPERVISOR_REVIEW_STREAM_KEY]: [
|
||||
{
|
||||
id: "1-0",
|
||||
message: {
|
||||
chatroomId: "room-1",
|
||||
packet: JSON.stringify({
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "low-confidence",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
summary: "unclear",
|
||||
userGoal: "help",
|
||||
}),
|
||||
queuedAt: "2026-05-18T00:01:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
userId: "user-1",
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
loadSupervisorOutcomes(
|
||||
garnet,
|
||||
new Date("2026-05-18T00:00:00.000Z"),
|
||||
new Date("2026-05-18T00:03:00.000Z"),
|
||||
),
|
||||
).resolves.toMatchObject([
|
||||
{
|
||||
reason: "low-confidence",
|
||||
resolution: {
|
||||
action: "manual-reply",
|
||||
},
|
||||
reviewId: "1-0",
|
||||
shadowEvalEntry: {
|
||||
request: {
|
||||
requestId: "request-1",
|
||||
},
|
||||
},
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,177 @@
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
parseSupervisorReviewEntry,
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
type SupervisorReviewAction,
|
||||
type SupervisorReviewResolution,
|
||||
} from "@skills/supervisor/review-queue";
|
||||
import { BACKGROUND_STREAM_KEY, type ShadowEvalEnvelope } from "./shadow-bus.js";
|
||||
|
||||
export interface SupervisorOutcome {
|
||||
queuedAt: string;
|
||||
reason: HandoffReason;
|
||||
resolution: {
|
||||
action: SupervisorReviewAction;
|
||||
resolvedAt: string;
|
||||
};
|
||||
reviewId: string;
|
||||
shadowEvalEntry?: ShadowEvalEnvelope | undefined;
|
||||
sourceChain: string[];
|
||||
}
|
||||
|
||||
function parseJson<T>(value: string | undefined): T | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(value) as T;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function isWithinIso(
|
||||
value: string,
|
||||
since: Date,
|
||||
until: Date,
|
||||
): boolean {
|
||||
const time = Date.parse(value);
|
||||
return (
|
||||
Number.isFinite(time) &&
|
||||
time >= since.getTime() &&
|
||||
time <= until.getTime()
|
||||
);
|
||||
}
|
||||
|
||||
function parseShadowEntry(
|
||||
entry: StreamEntry,
|
||||
): ShadowEvalEnvelope | null {
|
||||
const input = parseJson<{ request?: ShadowEvalEnvelope["request"] }>(
|
||||
entry.message["input"],
|
||||
);
|
||||
const response = parseJson<ShadowEvalEnvelope["response"]>(
|
||||
entry.message["output"],
|
||||
);
|
||||
const skill = entry.message["skill"];
|
||||
const createdAt = entry.message["createdAt"];
|
||||
if (!input?.request || !response || !skill || !createdAt) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
classified: parseJson(entry.message["classified"]) ?? {},
|
||||
createdAt,
|
||||
derived: parseJson(entry.message["derived"]) ?? {},
|
||||
request: input.request,
|
||||
response,
|
||||
skill,
|
||||
...(entry.message["latencyMs"]
|
||||
? { latencyMs: Number(entry.message["latencyMs"]) }
|
||||
: {}),
|
||||
...(entry.message["process"]
|
||||
? { process: parseJson(entry.message["process"]) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function isSupervisorReviewAction(
|
||||
value: string | undefined,
|
||||
): value is SupervisorReviewAction {
|
||||
return value === "manual-reply" || value === "reroute" || value === "dismiss";
|
||||
}
|
||||
|
||||
function parseResolutionEntry(
|
||||
entry: StreamEntry,
|
||||
): { resolution: SupervisorReviewResolution; reviewId: string } | null {
|
||||
const reviewId = entry.message["reviewId"];
|
||||
const resolvedAt = entry.message["resolvedAt"];
|
||||
const resolvedBy = entry.message["resolvedBy"];
|
||||
const action = entry.message["action"];
|
||||
if (
|
||||
!reviewId ||
|
||||
!resolvedAt ||
|
||||
!resolvedBy ||
|
||||
!isSupervisorReviewAction(action)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
resolution: {
|
||||
action,
|
||||
resolvedAt,
|
||||
resolvedBy,
|
||||
...(entry.message["note"] ? { note: entry.message["note"] } : {}),
|
||||
},
|
||||
reviewId,
|
||||
};
|
||||
}
|
||||
|
||||
function latestResolutionsByReviewId(
|
||||
entries: StreamEntry[],
|
||||
): Map<string, SupervisorReviewResolution> {
|
||||
const byReviewId = new Map<string, SupervisorReviewResolution>();
|
||||
for (const entry of entries) {
|
||||
const parsed = parseResolutionEntry(entry);
|
||||
if (!parsed) {
|
||||
continue;
|
||||
}
|
||||
const current = byReviewId.get(parsed.reviewId);
|
||||
if (!current || parsed.resolution.resolvedAt > current.resolvedAt) {
|
||||
byReviewId.set(parsed.reviewId, parsed.resolution);
|
||||
}
|
||||
}
|
||||
return byReviewId;
|
||||
}
|
||||
|
||||
export async function loadSupervisorOutcomes(
|
||||
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||
since: Date,
|
||||
until: Date,
|
||||
): Promise<SupervisorOutcome[]> {
|
||||
const [reviewEntries, resolutionEntries, shadowEntries] = await Promise.all([
|
||||
garnetClient.readStreamRange(SUPERVISOR_REVIEW_STREAM_KEY, "-", "+"),
|
||||
garnetClient.readStreamRange(SUPERVISOR_RESOLUTIONS_STREAM_KEY, "-", "+"),
|
||||
garnetClient.readStreamRange(BACKGROUND_STREAM_KEY, "-", "+"),
|
||||
]);
|
||||
const shadowByRequestId = new Map(
|
||||
shadowEntries.flatMap((entry) => {
|
||||
const shadow = parseShadowEntry(entry);
|
||||
return shadow ? [[shadow.request.requestId, shadow] as const] : [];
|
||||
}),
|
||||
);
|
||||
const resolutionByReviewId = latestResolutionsByReviewId(resolutionEntries);
|
||||
const outcomes: SupervisorOutcome[] = [];
|
||||
|
||||
for (const entry of reviewEntries) {
|
||||
const review = parseSupervisorReviewEntry(entry);
|
||||
if (!review || !isWithinIso(review.queuedAt, since, until)) {
|
||||
continue;
|
||||
}
|
||||
const resolution = resolutionByReviewId.get(review.id);
|
||||
if (!resolution || !isWithinIso(resolution.resolvedAt, since, until)) {
|
||||
continue;
|
||||
}
|
||||
outcomes.push({
|
||||
queuedAt: review.queuedAt,
|
||||
reason: review.reason,
|
||||
resolution: {
|
||||
action: resolution.action,
|
||||
resolvedAt: resolution.resolvedAt,
|
||||
},
|
||||
reviewId: review.id,
|
||||
shadowEvalEntry: shadowByRequestId.get(review.requestId),
|
||||
sourceChain: review.packet?.sourceChain ?? [],
|
||||
});
|
||||
}
|
||||
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
export const ROUTER_TRAINING_SIGNAL_STREAMS = {
|
||||
background: BACKGROUND_STREAM_KEY,
|
||||
resolutions: SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
reviews: SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
} as const;
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
getShadowEvalSamplingRate,
|
||||
} from "../config.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
export const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
|
||||
export interface ShadowEvalEnvelope {
|
||||
classified: unknown;
|
||||
|
||||
@@ -262,6 +262,7 @@ async function bootstrap(): Promise<void> {
|
||||
natsClient,
|
||||
roomContextService,
|
||||
shadowEvalBus,
|
||||
skillRegistry,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
+34
-24
@@ -25,31 +25,41 @@ describe("operations review CLI formatters", () => {
|
||||
|
||||
it("formats packet details for show output", () => {
|
||||
expect(
|
||||
formatReviewShow({
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
packet: {
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [
|
||||
{
|
||||
id: "e1",
|
||||
kind: "message",
|
||||
summary: "unclear message",
|
||||
},
|
||||
],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "low-confidence",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
summary: "needs review",
|
||||
userGoal: "의도 확인",
|
||||
formatReviewShow(
|
||||
{
|
||||
chatroomId: "room-1",
|
||||
id: "1-0",
|
||||
packet: {
|
||||
constraints: [],
|
||||
currentState: {},
|
||||
evidence: [
|
||||
{
|
||||
id: "e1",
|
||||
kind: "message",
|
||||
summary: "unclear message",
|
||||
},
|
||||
],
|
||||
expectedOutputSchema: "text",
|
||||
handoffReason: "low-confidence",
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
summary: "needs review",
|
||||
userGoal: "의도 확인",
|
||||
},
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
userId: "user-1",
|
||||
},
|
||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||
reason: "low-confidence",
|
||||
requestId: "request-1",
|
||||
target: "human-review",
|
||||
userId: "user-1",
|
||||
}),
|
||||
[
|
||||
{
|
||||
action: "manual-reply",
|
||||
note: "handled",
|
||||
resolvedAt: "2026-05-18T01:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
},
|
||||
],
|
||||
),
|
||||
).toContain("sourceChain: onboarding > supervisor");
|
||||
});
|
||||
|
||||
|
||||
+23
-3
@@ -5,12 +5,14 @@ import { loadEnvironmentFromNodeEnv } from "@common/env-loader";
|
||||
import { connectGarnetForCli } from "@common/garnet-cli";
|
||||
import { initLogger } from "@obs/logger";
|
||||
import {
|
||||
getResolutionHistory,
|
||||
getSupervisorReview,
|
||||
getSupervisorReviewStats,
|
||||
listSupervisorReview,
|
||||
resolveSupervisorReview,
|
||||
type SupervisorReviewAction,
|
||||
type SupervisorReviewItem,
|
||||
type SupervisorReviewResolution,
|
||||
type SupervisorReviewStats,
|
||||
} from "@skills/supervisor/review-queue";
|
||||
import {
|
||||
@@ -95,12 +97,24 @@ export function formatReviewList(items: SupervisorReviewItem[]): string {
|
||||
return rows.map((row) => row.join("\t")).join("\n");
|
||||
}
|
||||
|
||||
export function formatReviewShow(item: SupervisorReviewItem): string {
|
||||
export function formatReviewShow(
|
||||
item: SupervisorReviewItem,
|
||||
history?: SupervisorReviewResolution[],
|
||||
): string {
|
||||
const packet = item.packet;
|
||||
const evidence =
|
||||
packet?.evidence
|
||||
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
|
||||
.join("\n") || "-";
|
||||
const historyLines =
|
||||
history?.map((resolution) =>
|
||||
[
|
||||
`${resolution.resolvedAt} ${resolution.action} by ${resolution.resolvedBy}`,
|
||||
resolution.note ? `note=${resolution.note}` : null,
|
||||
]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join(" "),
|
||||
) ?? [];
|
||||
|
||||
return [
|
||||
`id: ${item.id}`,
|
||||
@@ -124,6 +138,9 @@ export function formatReviewShow(item: SupervisorReviewItem): string {
|
||||
`note: ${stringifyOptional(item.resolution.note)}`,
|
||||
].join("\n")
|
||||
: "[resolution]\n-",
|
||||
...(history
|
||||
? ["[history]", ...(historyLines.length > 0 ? historyLines : ["-"])]
|
||||
: []),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -191,10 +208,13 @@ async function handleReviewShow(
|
||||
console.error(`[review] entry not found: ${id}`);
|
||||
return;
|
||||
}
|
||||
const history = hasFlag(flags, "history")
|
||||
? await getResolutionHistory(connection.garnetClient, id)
|
||||
: undefined;
|
||||
console.info(
|
||||
hasFlag(flags, "json")
|
||||
? JSON.stringify(item, null, 2)
|
||||
: formatReviewShow(item),
|
||||
? JSON.stringify({ ...item, ...(history ? { history } : {}) }, null, 2)
|
||||
: formatReviewShow(item, history),
|
||||
);
|
||||
} finally {
|
||||
await connection.close();
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { ShadowEvalBus } from "@eval/shadow-bus";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { RoomContextService } from "@room/room-context-service";
|
||||
import type { Skill } from "./skill.js";
|
||||
import type { SkillRegistry } from "./skill-loader.js";
|
||||
|
||||
export interface SkillDeps {
|
||||
garnetClient: GarnetClient;
|
||||
@@ -11,6 +12,7 @@ export interface SkillDeps {
|
||||
natsClient: NatsClient;
|
||||
roomContextService?: RoomContextService | undefined;
|
||||
shadowEvalBus: ShadowEvalBus;
|
||||
skillRegistry?: SkillRegistry | undefined;
|
||||
}
|
||||
|
||||
export type SkillFactory = (deps: SkillDeps) => Skill;
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: supervisor
|
||||
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
|
||||
description: LLM 기반 root cause 라우팅 supervisor.
|
||||
version: 1
|
||||
priority: 10
|
||||
domain: meta
|
||||
@@ -19,4 +19,4 @@ resources:
|
||||
trustTier: T4
|
||||
---
|
||||
|
||||
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||
LLM 기반 root cause 분석과 규칙 fallback을 함께 사용하는 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
|
||||
|
||||
function formatSkill(manifest: SkillManifest): string {
|
||||
return [
|
||||
`- ${manifest.name}: ${manifest.description}`,
|
||||
`domain=${manifest.domain}`,
|
||||
`canHandle=${manifest.canHandle.join(", ") || "*"}`,
|
||||
`cannotHandle=${manifest.cannotHandle.join(", ") || "-"}`,
|
||||
`risk=${manifest.riskTier}`,
|
||||
].join(" | ");
|
||||
}
|
||||
|
||||
export function buildSupervisorSystemPrompt(
|
||||
registry: SkillRegistry,
|
||||
_packet?: HandoffPacket,
|
||||
): string {
|
||||
const skills = registry
|
||||
.list()
|
||||
.filter((manifest) => manifest.name !== "supervisor");
|
||||
return [
|
||||
"너는 multi-agent 시스템의 supervisor다.",
|
||||
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
|
||||
"",
|
||||
"[사용 가능한 specialist]",
|
||||
...skills.map(formatSkill),
|
||||
"",
|
||||
"[응답 형식]",
|
||||
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단"}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildSupervisorUserPrompt(
|
||||
packet?: HandoffPacket,
|
||||
): string {
|
||||
if (!packet) {
|
||||
return "사용자 메시지가 명확하지 않음. 어디로 보낼지 결정해줘.";
|
||||
}
|
||||
|
||||
return [
|
||||
`[escalation reason]: ${packet.handoffReason}`,
|
||||
`[user goal]: ${packet.userGoal}`,
|
||||
`[summary]: ${packet.summary}`,
|
||||
`[source chain]: ${packet.sourceChain.join(" -> ")}`,
|
||||
].join("\n");
|
||||
}
|
||||
@@ -5,5 +5,10 @@ import { SupervisorSkill } from "./skill.js";
|
||||
export { SupervisorSkill } from "./skill.js";
|
||||
|
||||
export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
|
||||
return new SupervisorSkill(
|
||||
deps.garnetClient,
|
||||
deps.natsClient,
|
||||
deps.llmClient,
|
||||
deps.skillRegistry,
|
||||
);
|
||||
}
|
||||
@@ -4,15 +4,18 @@ import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorReview,
|
||||
getSupervisorReviewStats,
|
||||
getResolutionHistory,
|
||||
listSupervisorReview,
|
||||
parseSupervisorReviewEntry,
|
||||
resolveSupervisorReview,
|
||||
SUPERVISOR_RESOLVED_HASH_PREFIX,
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||
} from "./review-queue.js";
|
||||
|
||||
class FakeGarnet {
|
||||
readonly hashes = new Map<string, Record<string, string>>();
|
||||
readonly resolutionStream: StreamEntry[] = [];
|
||||
readonly stream: StreamEntry[];
|
||||
|
||||
constructor(stream: StreamEntry[]) {
|
||||
@@ -25,8 +28,12 @@ class FakeGarnet {
|
||||
end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
|
||||
const entries = this.stream.filter((entry) => {
|
||||
expect([SUPERVISOR_REVIEW_STREAM_KEY, SUPERVISOR_RESOLUTIONS_STREAM_KEY]).toContain(key);
|
||||
const source =
|
||||
key === SUPERVISOR_RESOLUTIONS_STREAM_KEY
|
||||
? this.resolutionStream
|
||||
: this.stream;
|
||||
const entries = source.filter((entry) => {
|
||||
const afterStart = start === "-" || entry.id >= start;
|
||||
const beforeEnd = end === "+" || entry.id <= end;
|
||||
return afterStart && beforeEnd;
|
||||
@@ -43,6 +50,16 @@ class FakeGarnet {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
appendStream(key: string, fields: Record<string, string>): Promise<string> {
|
||||
expect(key).toBe(SUPERVISOR_RESOLUTIONS_STREAM_KEY);
|
||||
const id = `${this.resolutionStream.length + 1}-0`;
|
||||
this.resolutionStream.push({
|
||||
id,
|
||||
message: { ...fields },
|
||||
});
|
||||
return Promise.resolve(id);
|
||||
}
|
||||
|
||||
expireKey(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
@@ -140,6 +157,35 @@ describe("supervisor review queue", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("dual-writes resolution updates to an audit stream", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
await resolveSupervisorReview(garnet, "1-0", {
|
||||
action: "manual-reply",
|
||||
note: "handled",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
});
|
||||
await resolveSupervisorReview(garnet, "1-0", {
|
||||
action: "dismiss",
|
||||
resolvedAt: "2026-05-18T03:00:00.000Z",
|
||||
resolvedBy: "operator-2@example.com",
|
||||
});
|
||||
|
||||
await expect(getResolutionHistory(garnet, "1-0")).resolves.toEqual([
|
||||
{
|
||||
action: "dismiss",
|
||||
resolvedAt: "2026-05-18T03:00:00.000Z",
|
||||
resolvedBy: "operator-2@example.com",
|
||||
},
|
||||
{
|
||||
action: "manual-reply",
|
||||
note: "handled",
|
||||
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||
resolvedBy: "operator@example.com",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("computes reason totals and top source chains", async () => {
|
||||
const garnet = new FakeGarnet(entries);
|
||||
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffReason } from "@agent/types";
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
getSupervisorResolutionsStreamMaxLen,
|
||||
getSupervisorReviewQueueMaxLen,
|
||||
getSupervisorReviewResolutionTtlSec,
|
||||
} from "../../config.js";
|
||||
@@ -9,6 +10,8 @@ import {
|
||||
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
|
||||
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
|
||||
"lolympus-agent:review-resolved:";
|
||||
export const SUPERVISOR_RESOLUTIONS_STREAM_KEY =
|
||||
"lolympus-agent:review-resolutions";
|
||||
|
||||
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
|
||||
|
||||
@@ -67,7 +70,7 @@ type SupervisorReviewListGarnetClient = Pick<
|
||||
|
||||
type SupervisorReviewResolveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"expireKey" | "setHash"
|
||||
"appendStream" | "expireKey" | "setHash"
|
||||
>;
|
||||
|
||||
const BASE_FIELDS = new Set([
|
||||
@@ -285,6 +288,36 @@ export async function resolveSupervisorReview(
|
||||
const key = resolutionKey(id);
|
||||
await garnetClient.setHash(key, fields);
|
||||
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
|
||||
await garnetClient.appendStream(
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
{
|
||||
...fields,
|
||||
reviewId: id,
|
||||
type: "resolution",
|
||||
},
|
||||
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
|
||||
);
|
||||
}
|
||||
|
||||
export async function getResolutionHistory(
|
||||
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||
id: string,
|
||||
): Promise<SupervisorReviewResolution[]> {
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||
"-",
|
||||
"+",
|
||||
);
|
||||
|
||||
return entries
|
||||
.flatMap((entry) => {
|
||||
if (entry.message["reviewId"] !== id) {
|
||||
return [];
|
||||
}
|
||||
const resolution = parseResolution(entry.message);
|
||||
return resolution ? [resolution] : [];
|
||||
})
|
||||
.sort((left, right) => right.resolvedAt.localeCompare(left.resolvedAt));
|
||||
}
|
||||
|
||||
export async function getSupervisorReviewStats(
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const supervisorRoutingSchema = z.object({
|
||||
confidence: z.number().min(0).max(1),
|
||||
reason: z.string().max(200),
|
||||
target: z.string().min(1),
|
||||
});
|
||||
|
||||
export type SupervisorRoutingDecision = z.infer<typeof supervisorRoutingSchema>;
|
||||
@@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest";
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { SkillRegistry } from "@skills/skill-loader";
|
||||
import { SupervisorSkill } from "./skill.js";
|
||||
|
||||
class FakeGarnet {
|
||||
@@ -27,6 +29,44 @@ class FakeNats {
|
||||
}
|
||||
}
|
||||
|
||||
class FakeLlm {
|
||||
chatCompletion(): Promise<{
|
||||
content: string;
|
||||
endpoint: string;
|
||||
message: { content: string; role: "assistant" };
|
||||
provider: "llama";
|
||||
raw: object;
|
||||
}> {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
confidence: 0.9,
|
||||
reason: "광고 규칙 문의라 moderation이 적합",
|
||||
target: "moderation",
|
||||
}),
|
||||
endpoint: "memory",
|
||||
message: { content: "", role: "assistant" },
|
||||
provider: "llama",
|
||||
raw: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const registry = {
|
||||
get: (name: string) => (name === "moderation" ? { name } : null),
|
||||
list: () => [
|
||||
{
|
||||
canHandle: ["rules"],
|
||||
cannotHandle: [],
|
||||
description: "커뮤니티 규칙 처리",
|
||||
domain: "community",
|
||||
handoffTargets: [],
|
||||
name: "moderation",
|
||||
outputContract: {},
|
||||
riskTier: "medium",
|
||||
},
|
||||
],
|
||||
} as unknown as SkillRegistry;
|
||||
|
||||
const baseRequest: AgentRequest = {
|
||||
chatroomId: "room-1",
|
||||
nickname: "tester",
|
||||
@@ -70,6 +110,7 @@ describe("SupervisorSkill", () => {
|
||||
|
||||
expect(response.text).toContain("운영자");
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "rules",
|
||||
supervisorReason: "max-hop",
|
||||
suggestedTarget: "onboarding",
|
||||
});
|
||||
@@ -96,4 +137,44 @@ describe("SupervisorSkill", () => {
|
||||
metric: "agent.supervisor_handoff",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses LLM routing when the decision is confident and valid", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const skill = new SupervisorSkill(
|
||||
garnet,
|
||||
nats,
|
||||
new FakeLlm() as unknown as Pick<LlmClient, "chatCompletion">,
|
||||
registry,
|
||||
);
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
confidence: 0.7,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "광고 규칙 문의",
|
||||
},
|
||||
request: baseRequest,
|
||||
response: sourceResponse,
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
});
|
||||
|
||||
const response = await skill.handle(
|
||||
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||
[],
|
||||
);
|
||||
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "llm",
|
||||
supervisorReason: "광고 규칙 문의라 moderation이 적합",
|
||||
suggestedTarget: "moderation",
|
||||
});
|
||||
expect(garnet.streams[0]?.fields).toMatchObject({
|
||||
reason: "out-of-scope",
|
||||
target: "moderation",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -4,14 +4,29 @@ import type {
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { AlertService } from "@obs/alert";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
||||
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import {
|
||||
getSupervisorLlmConfidenceThreshold,
|
||||
getSupervisorLlmEnabled,
|
||||
} from "../../config.js";
|
||||
import {
|
||||
buildSupervisorSystemPrompt,
|
||||
buildSupervisorUserPrompt,
|
||||
} from "./capability-prompt.js";
|
||||
import { appendSupervisorReview } from "./review-queue.js";
|
||||
import { routeSupervisorTarget } from "./routing-rules.js";
|
||||
import {
|
||||
supervisorRoutingSchema,
|
||||
type SupervisorRoutingDecision,
|
||||
} from "./schema.js";
|
||||
|
||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||
canHandle: ["*"],
|
||||
@@ -34,6 +49,8 @@ export class SupervisorSkill implements Skill {
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||
natsClient: Pick<NatsClient, "publish">,
|
||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||
) {
|
||||
this.alertService = new AlertService(garnetClient, natsClient);
|
||||
}
|
||||
@@ -51,7 +68,11 @@ export class SupervisorSkill implements Skill {
|
||||
_conversationHistory: ConversationMessage[],
|
||||
): Promise<AgentResponse> {
|
||||
const packet = readHandoffPacket(request);
|
||||
const target = routeSupervisorTarget(packet);
|
||||
const llmDecision = await this.routeByLlm(packet);
|
||||
const useLlmRoute =
|
||||
llmDecision !== null &&
|
||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
await appendSupervisorReview(this.garnetClient, {
|
||||
@@ -83,7 +104,8 @@ export class SupervisorSkill implements Skill {
|
||||
return {
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
supervisorReason: reason,
|
||||
routedBy: useLlmRoute ? "llm" : "rules",
|
||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||
suggestedTarget: target,
|
||||
},
|
||||
requestId: request.requestId,
|
||||
@@ -91,4 +113,54 @@ export class SupervisorSkill implements Skill {
|
||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||
};
|
||||
}
|
||||
|
||||
private async routeByLlm(
|
||||
packet?: HandoffPacket,
|
||||
): Promise<SupervisorRoutingDecision | null> {
|
||||
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
|
||||
role: "system",
|
||||
},
|
||||
{
|
||||
content: buildSupervisorUserPrompt(packet),
|
||||
role: "user",
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: 150,
|
||||
temperature: 0,
|
||||
tier: 0,
|
||||
},
|
||||
);
|
||||
if (!result?.content.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = supervisorRoutingSchema.safeParse(
|
||||
JSON.parse(result.content) as unknown,
|
||||
);
|
||||
if (!parsed.success || !this.isKnownTarget(parsed.data.target)) {
|
||||
return null;
|
||||
}
|
||||
return parsed.data;
|
||||
} catch (error) {
|
||||
getLogger().warn(
|
||||
{ err: error },
|
||||
"[Supervisor] LLM routing failed, falling back to rules",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private isKnownTarget(target: string): boolean {
|
||||
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user