Unify dispatch and add supervisor routing signals
This commit is contained in:
1 parent
f0b00e5fd2
commit
b63985694a
19 files changed
+1345
-812
No files matched your search
+37
-438
@@ -3,22 +3,14 @@ import type { ConversationStore } from "@conv/conversation-store";
|
|||||||
import type { NatsClient } from "@common/nats";
|
import type { NatsClient } from "@common/nats";
|
||||||
import { getRequestLogger, runWithContext } from "@obs/context";
|
import { getRequestLogger, runWithContext } from "@obs/context";
|
||||||
import { getLogger } from "@obs/logger";
|
import { getLogger } from "@obs/logger";
|
||||||
import { withSkillSpan, withSpan } from "@obs/spans";
|
import { withSpan } from "@obs/spans";
|
||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
import type { TransitionMatrix } from "@skills/transition-matrix";
|
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||||
import { parseControlTokens } from "@skills/control-token-parser";
|
|
||||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
|
||||||
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
|
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
|
||||||
import { metrics } from "@obs/metrics";
|
import { metrics } from "@obs/metrics";
|
||||||
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
|
import { BudgetTracker } from "./budget.js";
|
||||||
import { buildHandoffPacket } from "./handoff-packet.js";
|
import { dispatch } from "./dispatch.js";
|
||||||
import type {
|
import type { AgentRequest, AgentResponse } from "./types.js";
|
||||||
AgentRequest,
|
|
||||||
AgentResponse,
|
|
||||||
ConversationMessage,
|
|
||||||
HandoffReason,
|
|
||||||
HandoffRequest,
|
|
||||||
} from "./types.js";
|
|
||||||
import {
|
import {
|
||||||
getAgentRuntimeVerifierEnabled,
|
getAgentRuntimeVerifierEnabled,
|
||||||
getAgentRequestTimeoutMs,
|
getAgentRequestTimeoutMs,
|
||||||
@@ -167,14 +159,39 @@ export class Agent {
|
|||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const budget = new BudgetTracker();
|
const budget = new BudgetTracker();
|
||||||
budget.recordVisit(skill.name);
|
budget.recordVisit(skill.name);
|
||||||
const response = await this.dispatch({
|
const response = await dispatch(
|
||||||
budget,
|
{
|
||||||
chain: [skill.name],
|
budget,
|
||||||
depth: 0,
|
chain: [skill.name],
|
||||||
history,
|
depth: 0,
|
||||||
request,
|
history,
|
||||||
skill,
|
request,
|
||||||
});
|
skill,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
getSkillJudgeConfig,
|
||||||
|
onHandoff: (event) => {
|
||||||
|
metrics.handoffTotal.inc({
|
||||||
|
from: event.from,
|
||||||
|
reason: event.reason,
|
||||||
|
to: event.to,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
onHandoffLoop: (event) => {
|
||||||
|
metrics.handoffLoopTotal.inc({ from: event.from });
|
||||||
|
},
|
||||||
|
onValidatorFail: (event) => {
|
||||||
|
metrics.validatorFailTotal.inc({
|
||||||
|
skill: event.skill,
|
||||||
|
stage: event.stage,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
selectSkillForRequest,
|
||||||
|
shouldApplyRuntimeVerifier: getAgentRuntimeVerifierEnabled,
|
||||||
|
skills: this.skills,
|
||||||
|
transitionMatrix: this.transitionMatrix,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
if (response.text.trim()) {
|
if (response.text.trim()) {
|
||||||
await this.conversationStore.appendMessage(
|
await this.conversationStore.appendMessage(
|
||||||
@@ -230,422 +247,4 @@ export class Agent {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async dispatch(input: {
|
|
||||||
budget: BudgetTracker;
|
|
||||||
chain: string[];
|
|
||||||
depth: number;
|
|
||||||
history: ConversationMessage[];
|
|
||||||
request: AgentRequest;
|
|
||||||
skill: Skill;
|
|
||||||
}): Promise<AgentResponse> {
|
|
||||||
const { budget, chain, depth, history, request, skill } = input;
|
|
||||||
const contextBase = {
|
|
||||||
chatroomId: request.chatroomId,
|
|
||||||
deadlineAt: new Date(
|
|
||||||
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
|
||||||
).toISOString(),
|
|
||||||
requestId: request.requestId,
|
|
||||||
skill: skill.name,
|
|
||||||
startedAt: new Date().toISOString(),
|
|
||||||
trigger: request.trigger,
|
|
||||||
userId: request.userId,
|
|
||||||
};
|
|
||||||
|
|
||||||
return runWithContext(
|
|
||||||
{
|
|
||||||
...contextBase,
|
|
||||||
budget: budget.snapshot(),
|
|
||||||
handoffChain: chain,
|
|
||||||
handoffDepth: depth,
|
|
||||||
},
|
|
||||||
async () => {
|
|
||||||
const response = await this.runSkillOnce({
|
|
||||||
chain,
|
|
||||||
depth,
|
|
||||||
history,
|
|
||||||
request,
|
|
||||||
skill,
|
|
||||||
});
|
|
||||||
const normalized = this.normalizeControlTokens(response, skill.name);
|
|
||||||
const verified = await this.applyRuntimeVerifier({
|
|
||||||
chain,
|
|
||||||
depth,
|
|
||||||
history,
|
|
||||||
request,
|
|
||||||
response: normalized,
|
|
||||||
skill,
|
|
||||||
});
|
|
||||||
const escalated = this.applyEscalationTrigger({
|
|
||||||
budget,
|
|
||||||
request,
|
|
||||||
response: verified,
|
|
||||||
skill,
|
|
||||||
});
|
|
||||||
const annotated = this.withHandoffMetadata(escalated, chain, depth);
|
|
||||||
|
|
||||||
const handoff = readHandoffRequest(annotated);
|
|
||||||
if (!handoff) {
|
|
||||||
return annotated;
|
|
||||||
}
|
|
||||||
|
|
||||||
const next = this.prepareHandoff({
|
|
||||||
budget,
|
|
||||||
chain,
|
|
||||||
currentSkill: skill.name,
|
|
||||||
depth,
|
|
||||||
handoff,
|
|
||||||
history,
|
|
||||||
request,
|
|
||||||
response: annotated,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!next) {
|
|
||||||
return this.withHandoffMetadata(
|
|
||||||
this.makeSupervisorFallback(request, handoff.reason),
|
|
||||||
[...chain, "supervisor"],
|
|
||||||
depth + 1,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.dispatch(next);
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private withHandoffMetadata(
|
|
||||||
response: AgentResponse,
|
|
||||||
chain: string[],
|
|
||||||
depth: number,
|
|
||||||
): AgentResponse {
|
|
||||||
return {
|
|
||||||
...response,
|
|
||||||
metadata: {
|
|
||||||
...(response.metadata ?? {}),
|
|
||||||
handoffChain: chain,
|
|
||||||
handoffDepth: depth,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private async runSkillOnce(input: {
|
|
||||||
chain: string[];
|
|
||||||
depth: number;
|
|
||||||
history: ConversationMessage[];
|
|
||||||
request: AgentRequest;
|
|
||||||
skill: Skill;
|
|
||||||
}): Promise<AgentResponse> {
|
|
||||||
return withSkillSpan(
|
|
||||||
input.skill.name,
|
|
||||||
{
|
|
||||||
"agent.handoff_chain": input.chain.join(">"),
|
|
||||||
"agent.handoff_depth": input.depth,
|
|
||||||
"agent.skill": input.skill.name,
|
|
||||||
"agent.trigger": input.request.trigger,
|
|
||||||
},
|
|
||||||
() => input.skill.handle(input.request, input.history),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private normalizeControlTokens(
|
|
||||||
response: AgentResponse,
|
|
||||||
currentSkill: string,
|
|
||||||
): AgentResponse {
|
|
||||||
const parsed = parseControlTokens(response.text);
|
|
||||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
|
||||||
const metadata = { ...(response.metadata ?? {}) };
|
|
||||||
if (handoff?.kind === "handoff") {
|
|
||||||
metadata.handoffRequest = {
|
|
||||||
confidence: response.confidence,
|
|
||||||
evidenceRefs: [],
|
|
||||||
expectedOutput: "text",
|
|
||||||
reason: handoff.reason,
|
|
||||||
risk: "medium",
|
|
||||||
targetAgent: handoff.targetAgent,
|
|
||||||
taskSpec: `Continue handling from ${currentSkill}`,
|
|
||||||
} satisfies HandoffRequest;
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
...response,
|
|
||||||
metadata,
|
|
||||||
text: parsed.stripped,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private async applyRuntimeVerifier(input: {
|
|
||||||
chain: string[];
|
|
||||||
depth: number;
|
|
||||||
history: ConversationMessage[];
|
|
||||||
request: AgentRequest;
|
|
||||||
response: AgentResponse;
|
|
||||||
skill: Skill;
|
|
||||||
}): Promise<AgentResponse> {
|
|
||||||
if (!getAgentRuntimeVerifierEnabled()) {
|
|
||||||
return input.response;
|
|
||||||
}
|
|
||||||
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
|
|
||||||
if (!verifier) {
|
|
||||||
return input.response;
|
|
||||||
}
|
|
||||||
const verdict = await verifier({
|
|
||||||
request: input.request,
|
|
||||||
response: input.response,
|
|
||||||
skill: input.skill.name,
|
|
||||||
});
|
|
||||||
if (verdict === "pass") {
|
|
||||||
return input.response;
|
|
||||||
}
|
|
||||||
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
|
|
||||||
if (verdict === "retry" && input.depth === 0) {
|
|
||||||
const retryResponse = await this.runSkillOnce({
|
|
||||||
...input,
|
|
||||||
request: {
|
|
||||||
...input.request,
|
|
||||||
context: {
|
|
||||||
...(input.request.context ?? {}),
|
|
||||||
runtimeVerifierRetry: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const normalizedRetry = this.normalizeControlTokens(
|
|
||||||
retryResponse,
|
|
||||||
input.skill.name,
|
|
||||||
);
|
|
||||||
if (readHandoffRequest(normalizedRetry)) {
|
|
||||||
return normalizedRetry;
|
|
||||||
}
|
|
||||||
const retryVerdict = await verifier({
|
|
||||||
request: input.request,
|
|
||||||
response: normalizedRetry,
|
|
||||||
skill: input.skill.name,
|
|
||||||
});
|
|
||||||
if (retryVerdict === "pass") {
|
|
||||||
return {
|
|
||||||
...normalizedRetry,
|
|
||||||
metadata: {
|
|
||||||
...(normalizedRetry.metadata ?? {}),
|
|
||||||
runtimeVerifier: "retry-pass",
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
metrics.validatorFailTotal.inc({
|
|
||||||
skill: input.skill.name,
|
|
||||||
stage: `retry-${retryVerdict}`,
|
|
||||||
});
|
|
||||||
return this.withRuntimeVerifierHandoff(normalizedRetry);
|
|
||||||
}
|
|
||||||
return this.withRuntimeVerifierHandoff(input.response);
|
|
||||||
}
|
|
||||||
|
|
||||||
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
|
||||||
return {
|
|
||||||
...response,
|
|
||||||
metadata: {
|
|
||||||
...(response.metadata ?? {}),
|
|
||||||
handoffRequest: {
|
|
||||||
confidence: response.confidence,
|
|
||||||
evidenceRefs: [],
|
|
||||||
expectedOutput: "text",
|
|
||||||
reason: "validation-failed",
|
|
||||||
risk: "medium",
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
taskSpec: "Runtime verifier rejected the response.",
|
|
||||||
} satisfies HandoffRequest,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private applyEscalationTrigger(input: {
|
|
||||||
budget: BudgetTracker;
|
|
||||||
request: AgentRequest;
|
|
||||||
response: AgentResponse;
|
|
||||||
skill: Skill;
|
|
||||||
}): AgentResponse {
|
|
||||||
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
|
||||||
return input.response;
|
|
||||||
}
|
|
||||||
|
|
||||||
const decision = decideEscalation({
|
|
||||||
budget: input.budget.snapshot(),
|
|
||||||
capabilityCard: input.skill.capabilityCard?.(),
|
|
||||||
classified: { confidence: input.response.confidence },
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
|
||||||
});
|
|
||||||
if (!decision.handoff) {
|
|
||||||
return input.response;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
...input.response,
|
|
||||||
metadata: {
|
|
||||||
...(input.response.metadata ?? {}),
|
|
||||||
handoffRequest: decision.handoff,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private prepareHandoff(input: {
|
|
||||||
budget: BudgetTracker;
|
|
||||||
chain: string[];
|
|
||||||
currentSkill: string;
|
|
||||||
depth: number;
|
|
||||||
handoff: HandoffRequest;
|
|
||||||
history: ConversationMessage[];
|
|
||||||
request: AgentRequest;
|
|
||||||
response: AgentResponse;
|
|
||||||
}):
|
|
||||||
| {
|
|
||||||
budget: BudgetTracker;
|
|
||||||
chain: string[];
|
|
||||||
depth: number;
|
|
||||||
history: ConversationMessage[];
|
|
||||||
request: AgentRequest;
|
|
||||||
skill: Skill;
|
|
||||||
}
|
|
||||||
| null {
|
|
||||||
const target = this.resolveHandoffTarget(input);
|
|
||||||
const nextSkill = selectSkillForRequest(this.skills, {
|
|
||||||
...input.request,
|
|
||||||
skill: target.targetAgent,
|
|
||||||
trigger: "handoff",
|
|
||||||
});
|
|
||||||
if (!nextSkill) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
input.budget.recordHandoff(nextSkill.name);
|
|
||||||
metrics.handoffTotal.inc({
|
|
||||||
from: input.currentSkill,
|
|
||||||
reason: target.reason,
|
|
||||||
to: nextSkill.name,
|
|
||||||
});
|
|
||||||
|
|
||||||
const nextChain = [...input.chain, nextSkill.name];
|
|
||||||
const packet = buildHandoffPacket({
|
|
||||||
handoff: {
|
|
||||||
...input.handoff,
|
|
||||||
reason: target.reason,
|
|
||||||
targetAgent: nextSkill.name,
|
|
||||||
},
|
|
||||||
history: input.history,
|
|
||||||
request: input.request,
|
|
||||||
response: input.response,
|
|
||||||
sourceChain: nextChain,
|
|
||||||
state: {
|
|
||||||
source: input.currentSkill,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
|
||||||
budget: input.budget,
|
|
||||||
chain: nextChain,
|
|
||||||
depth: input.depth + 1,
|
|
||||||
history: input.history,
|
|
||||||
request: {
|
|
||||||
...input.request,
|
|
||||||
context: {
|
|
||||||
...(input.request.context ?? {}),
|
|
||||||
handoffPacket: packet,
|
|
||||||
},
|
|
||||||
skill: nextSkill.name,
|
|
||||||
trigger: "handoff",
|
|
||||||
},
|
|
||||||
skill: nextSkill,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private resolveHandoffTarget(input: {
|
|
||||||
budget: BudgetTracker;
|
|
||||||
currentSkill: string;
|
|
||||||
handoff: HandoffRequest;
|
|
||||||
}): { reason: HandoffReason; targetAgent: string } {
|
|
||||||
const requestedTarget = input.handoff.targetAgent;
|
|
||||||
const allowed = this.transitionMatrix?.isAllowed(
|
|
||||||
input.currentSkill,
|
|
||||||
requestedTarget,
|
|
||||||
) ?? true;
|
|
||||||
if (!allowed) {
|
|
||||||
return {
|
|
||||||
reason: "policy-risk",
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
|
||||||
if (verdict === "ok") {
|
|
||||||
return {
|
|
||||||
reason: input.handoff.reason,
|
|
||||||
targetAgent: requestedTarget,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
if (verdict === "same-agent-loop") {
|
|
||||||
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
reason: budgetVerdictToReason(verdict),
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private makeSupervisorFallback(
|
|
||||||
request: AgentRequest,
|
|
||||||
reason: HandoffReason,
|
|
||||||
): AgentResponse {
|
|
||||||
return {
|
|
||||||
confidence: 0.4,
|
|
||||||
metadata: {
|
|
||||||
supervisorFallbackReason: reason,
|
|
||||||
},
|
|
||||||
requestId: request.requestId,
|
|
||||||
source: "template",
|
|
||||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
|
||||||
if (verdict === "same-agent-loop") {
|
|
||||||
return "same-agent-revisit";
|
|
||||||
}
|
|
||||||
if (verdict === "max-hops") {
|
|
||||||
return "max-hop";
|
|
||||||
}
|
|
||||||
return "policy-risk";
|
|
||||||
}
|
|
||||||
|
|
||||||
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
|
||||||
const value = response.metadata?.["handoffRequest"];
|
|
||||||
if (!value || typeof value !== "object") {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const handoff = value as Partial<HandoffRequest>;
|
|
||||||
if (
|
|
||||||
typeof handoff.targetAgent !== "string" ||
|
|
||||||
typeof handoff.reason !== "string" ||
|
|
||||||
typeof handoff.taskSpec !== "string" ||
|
|
||||||
typeof handoff.confidence !== "number" ||
|
|
||||||
typeof handoff.expectedOutput !== "string"
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
confidence: handoff.confidence,
|
|
||||||
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
|
||||||
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
|
||||||
: [],
|
|
||||||
expectedOutput: handoff.expectedOutput,
|
|
||||||
reason: handoff.reason,
|
|
||||||
risk:
|
|
||||||
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
|
||||||
? handoff.risk
|
|
||||||
: "medium",
|
|
||||||
targetAgent: handoff.targetAgent,
|
|
||||||
taskSpec: handoff.taskSpec,
|
|
||||||
...(handoff.userVisibleMessage
|
|
||||||
? { userVisibleMessage: handoff.userVisibleMessage }
|
|
||||||
: {}),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,581 @@
|
|||||||
|
import { getSkillJudgeConfig as defaultGetSkillJudgeConfig } from "@eval/judges/skill-config";
|
||||||
|
import type { SkillJudgeConfig } from "@eval/judges/skill-config";
|
||||||
|
import { runWithContext } from "@obs/context";
|
||||||
|
import { withSkillSpan } from "@obs/spans";
|
||||||
|
import { parseControlTokens } from "@skills/control-token-parser";
|
||||||
|
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
||||||
|
import type { Skill } from "@skills/skill";
|
||||||
|
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||||
|
import { getAgentRequestTimeoutMs, getSafetyMarginMs } from "../config.js";
|
||||||
|
import type { BudgetTracker, BudgetVerdict } from "./budget.js";
|
||||||
|
import { buildHandoffPacket } from "./handoff-packet.js";
|
||||||
|
import type {
|
||||||
|
AgentRequest,
|
||||||
|
AgentResponse,
|
||||||
|
ConversationMessage,
|
||||||
|
HandoffReason,
|
||||||
|
HandoffRequest,
|
||||||
|
} from "./types.js";
|
||||||
|
|
||||||
|
export interface HandoffEvent {
|
||||||
|
from: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
to: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ValidatorFailEvent {
|
||||||
|
skill: string;
|
||||||
|
stage: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EvalDispatchPlanStep {
|
||||||
|
confidence?: number | undefined;
|
||||||
|
expectedOutput?: string | undefined;
|
||||||
|
from: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
risk?: HandoffRequest["risk"] | undefined;
|
||||||
|
taskSpec?: string | undefined;
|
||||||
|
to: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EvalDispatchPlan {
|
||||||
|
maxDepth?: number | undefined;
|
||||||
|
steps: EvalDispatchPlanStep[];
|
||||||
|
useActualSkills?: boolean | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DispatchDependencies {
|
||||||
|
evalPlan?: EvalDispatchPlan | undefined;
|
||||||
|
getSkillJudgeConfig?: ((skillName: string) => SkillJudgeConfig) | undefined;
|
||||||
|
onHandoff?: ((event: HandoffEvent) => void) | undefined;
|
||||||
|
onHandoffLoop?: ((event: { from: string }) => void) | undefined;
|
||||||
|
onValidatorFail?: ((event: ValidatorFailEvent) => void) | undefined;
|
||||||
|
selectSkillForRequest: (
|
||||||
|
skills: readonly Skill[],
|
||||||
|
request: AgentRequest,
|
||||||
|
) => Skill | null;
|
||||||
|
shouldApplyRuntimeVerifier: () => boolean;
|
||||||
|
skills: readonly Skill[];
|
||||||
|
transitionMatrix?: TransitionMatrix | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DispatchInput {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
chain: string[];
|
||||||
|
depth: number;
|
||||||
|
evalPlanIndex?: number | undefined;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
lastHandoff?: HandoffRequest | undefined;
|
||||||
|
request: AgentRequest;
|
||||||
|
skill: Skill;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function dispatch(
|
||||||
|
input: DispatchInput,
|
||||||
|
deps: DispatchDependencies,
|
||||||
|
): Promise<AgentResponse> {
|
||||||
|
const { budget, chain, depth, history, lastHandoff, request, skill } = input;
|
||||||
|
const contextBase = {
|
||||||
|
chatroomId: request.chatroomId,
|
||||||
|
deadlineAt: new Date(
|
||||||
|
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
||||||
|
).toISOString(),
|
||||||
|
requestId: request.requestId,
|
||||||
|
skill: skill.name,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
trigger: request.trigger,
|
||||||
|
userId: request.userId,
|
||||||
|
};
|
||||||
|
|
||||||
|
return runWithContext(
|
||||||
|
{
|
||||||
|
...contextBase,
|
||||||
|
budget: budget.snapshot(),
|
||||||
|
handoffChain: chain,
|
||||||
|
handoffDepth: depth,
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
const planStep = getCurrentPlanStep(input, deps);
|
||||||
|
const response = planStep
|
||||||
|
? plannedHandoffResponse(request, planStep)
|
||||||
|
: deps.evalPlan && deps.evalPlan.useActualSkills !== true
|
||||||
|
? templateEvalResponse(request, skill.name)
|
||||||
|
: await runSkillOnce({
|
||||||
|
chain,
|
||||||
|
depth,
|
||||||
|
history,
|
||||||
|
request,
|
||||||
|
skill,
|
||||||
|
});
|
||||||
|
const normalized = normalizeControlTokens(response, skill.name);
|
||||||
|
const verified = await applyRuntimeVerifier(
|
||||||
|
{
|
||||||
|
chain,
|
||||||
|
depth,
|
||||||
|
history,
|
||||||
|
request,
|
||||||
|
response: normalized,
|
||||||
|
skill,
|
||||||
|
},
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
const escalated = applyEscalationTrigger({
|
||||||
|
budget,
|
||||||
|
request,
|
||||||
|
response: verified,
|
||||||
|
skill,
|
||||||
|
});
|
||||||
|
const annotated = withHandoffMetadata(
|
||||||
|
escalated,
|
||||||
|
chain,
|
||||||
|
depth,
|
||||||
|
lastHandoff,
|
||||||
|
);
|
||||||
|
|
||||||
|
const handoff = readHandoffRequest(annotated);
|
||||||
|
if (!handoff) {
|
||||||
|
return annotated;
|
||||||
|
}
|
||||||
|
|
||||||
|
const next = prepareHandoff(
|
||||||
|
{
|
||||||
|
budget,
|
||||||
|
chain,
|
||||||
|
currentSkill: skill.name,
|
||||||
|
depth,
|
||||||
|
evalPlanIndex: input.evalPlanIndex ?? 0,
|
||||||
|
handoff,
|
||||||
|
history,
|
||||||
|
planStepUsed: Boolean(planStep),
|
||||||
|
request,
|
||||||
|
response: annotated,
|
||||||
|
},
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!next) {
|
||||||
|
const fallbackHandoff: HandoffRequest = {
|
||||||
|
...handoff,
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
};
|
||||||
|
return withHandoffMetadata(
|
||||||
|
makeSupervisorFallback(request, handoff.reason),
|
||||||
|
[...chain, "supervisor"],
|
||||||
|
depth + 1,
|
||||||
|
fallbackHandoff,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxDepth = deps.evalPlan?.maxDepth ?? 8;
|
||||||
|
if (next.depth > maxDepth) {
|
||||||
|
const fallbackHandoff: HandoffRequest = {
|
||||||
|
confidence: 0.4,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "max-hop",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "Dispatch exceeded max loop depth.",
|
||||||
|
};
|
||||||
|
const fallbackChain =
|
||||||
|
chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"];
|
||||||
|
return withHandoffMetadata(
|
||||||
|
makeSupervisorFallback(request, "max-hop"),
|
||||||
|
fallbackChain,
|
||||||
|
fallbackChain.length - 1,
|
||||||
|
fallbackHandoff,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return dispatch(next, deps);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function withHandoffMetadata(
|
||||||
|
response: AgentResponse,
|
||||||
|
chain: string[],
|
||||||
|
depth: number,
|
||||||
|
lastHandoff?: HandoffRequest,
|
||||||
|
): AgentResponse {
|
||||||
|
return {
|
||||||
|
...response,
|
||||||
|
metadata: {
|
||||||
|
...(response.metadata ?? {}),
|
||||||
|
handoffChain: chain,
|
||||||
|
handoffDepth: depth,
|
||||||
|
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeControlTokens(
|
||||||
|
response: AgentResponse,
|
||||||
|
currentSkill: string,
|
||||||
|
): AgentResponse {
|
||||||
|
const parsed = parseControlTokens(response.text);
|
||||||
|
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
||||||
|
const metadata = { ...(response.metadata ?? {}) };
|
||||||
|
if (handoff?.kind === "handoff") {
|
||||||
|
metadata.handoffRequest = {
|
||||||
|
confidence: response.confidence,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: handoff.reason,
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: handoff.targetAgent,
|
||||||
|
taskSpec: `Continue handling from ${currentSkill}`,
|
||||||
|
} satisfies HandoffRequest;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...response,
|
||||||
|
metadata,
|
||||||
|
text: parsed.stripped,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readHandoffRequest(
|
||||||
|
response: AgentResponse,
|
||||||
|
): HandoffRequest | null {
|
||||||
|
const value = response.metadata?.["handoffRequest"];
|
||||||
|
if (!value || typeof value !== "object") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const handoff = value as Partial<HandoffRequest>;
|
||||||
|
if (
|
||||||
|
typeof handoff.targetAgent !== "string" ||
|
||||||
|
typeof handoff.reason !== "string" ||
|
||||||
|
typeof handoff.taskSpec !== "string" ||
|
||||||
|
typeof handoff.confidence !== "number" ||
|
||||||
|
typeof handoff.expectedOutput !== "string"
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
confidence: handoff.confidence,
|
||||||
|
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
||||||
|
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
||||||
|
: [],
|
||||||
|
expectedOutput: handoff.expectedOutput,
|
||||||
|
reason: handoff.reason,
|
||||||
|
risk:
|
||||||
|
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
||||||
|
? handoff.risk
|
||||||
|
: "medium",
|
||||||
|
targetAgent: handoff.targetAgent,
|
||||||
|
taskSpec: handoff.taskSpec,
|
||||||
|
...(handoff.userVisibleMessage
|
||||||
|
? { userVisibleMessage: handoff.userVisibleMessage }
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
||||||
|
if (verdict === "same-agent-loop") {
|
||||||
|
return "same-agent-revisit";
|
||||||
|
}
|
||||||
|
if (verdict === "max-hops") {
|
||||||
|
return "max-hop";
|
||||||
|
}
|
||||||
|
return "policy-risk";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runSkillOnce(input: {
|
||||||
|
chain: string[];
|
||||||
|
depth: number;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
request: AgentRequest;
|
||||||
|
skill: Skill;
|
||||||
|
}): Promise<AgentResponse> {
|
||||||
|
return withSkillSpan(
|
||||||
|
input.skill.name,
|
||||||
|
{
|
||||||
|
"agent.handoff_chain": input.chain.join(">"),
|
||||||
|
"agent.handoff_depth": input.depth,
|
||||||
|
"agent.skill": input.skill.name,
|
||||||
|
"agent.trigger": input.request.trigger,
|
||||||
|
},
|
||||||
|
() => input.skill.handle(input.request, input.history),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function applyRuntimeVerifier(
|
||||||
|
input: {
|
||||||
|
chain: string[];
|
||||||
|
depth: number;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
skill: Skill;
|
||||||
|
},
|
||||||
|
deps: DispatchDependencies,
|
||||||
|
): Promise<AgentResponse> {
|
||||||
|
if (!deps.shouldApplyRuntimeVerifier()) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
const judgeConfig = deps.getSkillJudgeConfig ?? defaultGetSkillJudgeConfig;
|
||||||
|
const verifier = judgeConfig(input.skill.name).runtimeVerifier;
|
||||||
|
if (!verifier) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
const verdict = await verifier({
|
||||||
|
request: input.request,
|
||||||
|
response: input.response,
|
||||||
|
skill: input.skill.name,
|
||||||
|
});
|
||||||
|
if (verdict === "pass") {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
deps.onValidatorFail?.({ skill: input.skill.name, stage: verdict });
|
||||||
|
if (verdict === "retry" && input.depth === 0) {
|
||||||
|
const retryResponse = await runSkillOnce({
|
||||||
|
...input,
|
||||||
|
request: {
|
||||||
|
...input.request,
|
||||||
|
context: {
|
||||||
|
...(input.request.context ?? {}),
|
||||||
|
runtimeVerifierRetry: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const normalizedRetry = normalizeControlTokens(
|
||||||
|
retryResponse,
|
||||||
|
input.skill.name,
|
||||||
|
);
|
||||||
|
if (readHandoffRequest(normalizedRetry)) {
|
||||||
|
return normalizedRetry;
|
||||||
|
}
|
||||||
|
const retryVerdict = await verifier({
|
||||||
|
request: input.request,
|
||||||
|
response: normalizedRetry,
|
||||||
|
skill: input.skill.name,
|
||||||
|
});
|
||||||
|
if (retryVerdict === "pass") {
|
||||||
|
return {
|
||||||
|
...normalizedRetry,
|
||||||
|
metadata: {
|
||||||
|
...(normalizedRetry.metadata ?? {}),
|
||||||
|
runtimeVerifier: "retry-pass",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
deps.onValidatorFail?.({
|
||||||
|
skill: input.skill.name,
|
||||||
|
stage: `retry-${retryVerdict}`,
|
||||||
|
});
|
||||||
|
return withRuntimeVerifierHandoff(normalizedRetry);
|
||||||
|
}
|
||||||
|
return withRuntimeVerifierHandoff(input.response);
|
||||||
|
}
|
||||||
|
|
||||||
|
function withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
||||||
|
return {
|
||||||
|
...response,
|
||||||
|
metadata: {
|
||||||
|
...(response.metadata ?? {}),
|
||||||
|
handoffRequest: {
|
||||||
|
confidence: response.confidence,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "validation-failed",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "Runtime verifier rejected the response.",
|
||||||
|
} satisfies HandoffRequest,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyEscalationTrigger(input: {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
skill: Skill;
|
||||||
|
}): AgentResponse {
|
||||||
|
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
const decision = decideEscalation({
|
||||||
|
budget: input.budget.snapshot(),
|
||||||
|
capabilityCard: input.skill.capabilityCard?.(),
|
||||||
|
classified: { confidence: input.response.confidence },
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
||||||
|
});
|
||||||
|
if (!decision.handoff) {
|
||||||
|
return input.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
...input.response,
|
||||||
|
metadata: {
|
||||||
|
...(input.response.metadata ?? {}),
|
||||||
|
handoffRequest: decision.handoff,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function prepareHandoff(
|
||||||
|
input: {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
chain: string[];
|
||||||
|
currentSkill: string;
|
||||||
|
depth: number;
|
||||||
|
evalPlanIndex: number;
|
||||||
|
handoff: HandoffRequest;
|
||||||
|
history: ConversationMessage[];
|
||||||
|
planStepUsed: boolean;
|
||||||
|
request: AgentRequest;
|
||||||
|
response: AgentResponse;
|
||||||
|
},
|
||||||
|
deps: DispatchDependencies,
|
||||||
|
): DispatchInput | null {
|
||||||
|
const target = resolveHandoffTarget(input, deps);
|
||||||
|
const nextSkill = deps.selectSkillForRequest(deps.skills, {
|
||||||
|
...input.request,
|
||||||
|
skill: target.targetAgent,
|
||||||
|
trigger: "handoff",
|
||||||
|
});
|
||||||
|
if (!nextSkill) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
input.budget.recordHandoff(nextSkill.name);
|
||||||
|
deps.onHandoff?.({
|
||||||
|
from: input.currentSkill,
|
||||||
|
reason: target.reason,
|
||||||
|
to: nextSkill.name,
|
||||||
|
});
|
||||||
|
|
||||||
|
const nextChain = [...input.chain, nextSkill.name];
|
||||||
|
const lastHandoff: HandoffRequest = {
|
||||||
|
...input.handoff,
|
||||||
|
reason: target.reason,
|
||||||
|
targetAgent: nextSkill.name,
|
||||||
|
};
|
||||||
|
const packet = buildHandoffPacket({
|
||||||
|
handoff: lastHandoff,
|
||||||
|
history: input.history,
|
||||||
|
request: input.request,
|
||||||
|
response: input.response,
|
||||||
|
sourceChain: nextChain,
|
||||||
|
state: {
|
||||||
|
source: input.currentSkill,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
budget: input.budget,
|
||||||
|
chain: nextChain,
|
||||||
|
depth: input.depth + 1,
|
||||||
|
evalPlanIndex: input.evalPlanIndex + (input.planStepUsed ? 1 : 0),
|
||||||
|
history: input.history,
|
||||||
|
lastHandoff,
|
||||||
|
request: {
|
||||||
|
...input.request,
|
||||||
|
context: {
|
||||||
|
...(input.request.context ?? {}),
|
||||||
|
handoffPacket: packet,
|
||||||
|
},
|
||||||
|
skill: nextSkill.name,
|
||||||
|
trigger: "handoff",
|
||||||
|
},
|
||||||
|
skill: nextSkill,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveHandoffTarget(
|
||||||
|
input: {
|
||||||
|
budget: BudgetTracker;
|
||||||
|
currentSkill: string;
|
||||||
|
handoff: HandoffRequest;
|
||||||
|
},
|
||||||
|
deps: DispatchDependencies,
|
||||||
|
): { reason: HandoffReason; targetAgent: string } {
|
||||||
|
const requestedTarget = input.handoff.targetAgent;
|
||||||
|
const allowed =
|
||||||
|
deps.transitionMatrix?.isAllowed(input.currentSkill, requestedTarget) ?? true;
|
||||||
|
if (!allowed) {
|
||||||
|
return {
|
||||||
|
reason: "policy-risk",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
||||||
|
if (verdict === "ok") {
|
||||||
|
return {
|
||||||
|
reason: input.handoff.reason,
|
||||||
|
targetAgent: requestedTarget,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (verdict === "same-agent-loop") {
|
||||||
|
deps.onHandoffLoop?.({ from: input.currentSkill });
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
reason: budgetVerdictToReason(verdict),
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeSupervisorFallback(
|
||||||
|
request: AgentRequest,
|
||||||
|
reason: HandoffReason,
|
||||||
|
): AgentResponse {
|
||||||
|
return {
|
||||||
|
confidence: 0.4,
|
||||||
|
metadata: {
|
||||||
|
supervisorFallbackReason: reason,
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getCurrentPlanStep(
|
||||||
|
input: DispatchInput,
|
||||||
|
deps: DispatchDependencies,
|
||||||
|
): EvalDispatchPlanStep | undefined {
|
||||||
|
const index = input.evalPlanIndex ?? 0;
|
||||||
|
const step = deps.evalPlan?.steps[index];
|
||||||
|
return step?.from === input.skill.name ? step : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function plannedHandoffResponse(
|
||||||
|
request: AgentRequest,
|
||||||
|
step: EvalDispatchPlanStep,
|
||||||
|
): AgentResponse {
|
||||||
|
return {
|
||||||
|
confidence: step.confidence ?? 0.72,
|
||||||
|
metadata: {
|
||||||
|
handoffRequest: {
|
||||||
|
confidence: step.confidence ?? 0.72,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: step.expectedOutput ?? "text",
|
||||||
|
reason: step.reason,
|
||||||
|
risk: step.risk ?? "medium",
|
||||||
|
targetAgent: step.to,
|
||||||
|
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
|
||||||
|
} satisfies HandoffRequest,
|
||||||
|
},
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: `handoff ${step.from} to ${step.to}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function templateEvalResponse(
|
||||||
|
request: AgentRequest,
|
||||||
|
skillName: string,
|
||||||
|
): AgentResponse {
|
||||||
|
return {
|
||||||
|
confidence: 1,
|
||||||
|
requestId: request.requestId,
|
||||||
|
source: "template",
|
||||||
|
text: `handoff eval final: ${skillName}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -229,6 +229,18 @@ export function getSupervisorReviewResolutionTtlSec(): number {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getSupervisorResolutionsStreamMaxLen(): number {
|
||||||
|
return getPositiveNumber("SUPERVISOR_RESOLUTIONS_STREAM_MAXLEN", 100_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getSupervisorLlmEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_SUPERVISOR_LLM_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getSupervisorLlmConfidenceThreshold(): number {
|
||||||
|
return getRatioNumber("AGENT_SUPERVISOR_LLM_CONFIDENCE_THRESHOLD", 0.6);
|
||||||
|
}
|
||||||
|
|
||||||
export function getEvalCalibrationMinCorr(): number {
|
export function getEvalCalibrationMinCorr(): number {
|
||||||
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
|
||||||
}
|
}
|
||||||
|
|||||||
+68
-337
@@ -1,16 +1,10 @@
|
|||||||
import { BudgetTracker, type BudgetVerdict } from "@agent/budget";
|
|
||||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
|
||||||
import { selectSkillForRequest } from "@agent/agent";
|
import { selectSkillForRequest } from "@agent/agent";
|
||||||
import type {
|
import { BudgetTracker } from "@agent/budget";
|
||||||
AgentRequest,
|
import { dispatch, type EvalDispatchPlan, type EvalDispatchPlanStep } from "@agent/dispatch";
|
||||||
AgentResponse,
|
import type { AgentRequest, AgentResponse, ConversationMessage, HandoffReason } from "@agent/types";
|
||||||
ConversationMessage,
|
|
||||||
HandoffReason,
|
|
||||||
HandoffRequest,
|
|
||||||
} from "@agent/types";
|
|
||||||
import { parseControlTokens } from "@skills/control-token-parser";
|
|
||||||
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
|
||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
|
import type { TransitionMatrix } from "@skills/transition-matrix";
|
||||||
|
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
||||||
|
|
||||||
export interface EvalDispatchResult {
|
export interface EvalDispatchResult {
|
||||||
chain: string[];
|
chain: string[];
|
||||||
@@ -18,29 +12,43 @@ export interface EvalDispatchResult {
|
|||||||
response: AgentResponse;
|
response: AgentResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface EvalHandoffPlanStep {
|
|
||||||
confidence?: number | undefined;
|
|
||||||
expectedOutput?: string | undefined;
|
|
||||||
from: string;
|
|
||||||
reason: HandoffReason;
|
|
||||||
risk?: HandoffRequest["risk"] | undefined;
|
|
||||||
taskSpec?: string | undefined;
|
|
||||||
to: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
function asRecord(value: unknown): Record<string, unknown> {
|
function asRecord(value: unknown): Record<string, unknown> {
|
||||||
return typeof value === "object" && value !== null
|
return typeof value === "object" && value !== null
|
||||||
? (value as Record<string, unknown>)
|
? (value as Record<string, unknown>)
|
||||||
: {};
|
: {};
|
||||||
}
|
}
|
||||||
|
|
||||||
function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
|
const HANDOFF_REASONS = [
|
||||||
|
"out-of-scope",
|
||||||
|
"missing-tool",
|
||||||
|
"low-confidence",
|
||||||
|
"validation-failed",
|
||||||
|
"same-agent-revisit",
|
||||||
|
"max-hop",
|
||||||
|
"high-risk-action",
|
||||||
|
"policy-risk",
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
function isHandoffReason(value: unknown): value is HandoffReason {
|
||||||
|
return HANDOFF_REASONS.includes(value as HandoffReason);
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPositiveNumber(
|
||||||
|
context: Record<string, unknown> | undefined,
|
||||||
|
key: string,
|
||||||
|
fallback: number,
|
||||||
|
): number {
|
||||||
|
const value = context?.[key];
|
||||||
|
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPlan(request: AgentRequest): EvalDispatchPlanStep[] {
|
||||||
const rawPlan = asRecord(request.context)["evalHandoffPlan"];
|
const rawPlan = asRecord(request.context)["evalHandoffPlan"];
|
||||||
if (!Array.isArray(rawPlan)) {
|
if (!Array.isArray(rawPlan)) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
return rawPlan.flatMap((item): EvalHandoffPlanStep[] => {
|
return rawPlan.flatMap((item): EvalDispatchPlanStep[] => {
|
||||||
const record = asRecord(item);
|
const record = asRecord(item);
|
||||||
const from = record["from"];
|
const from = record["from"];
|
||||||
const to = record["to"];
|
const to = record["to"];
|
||||||
@@ -55,14 +63,8 @@ function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
|
|||||||
|
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
confidence:
|
confidence: typeof record["confidence"] === "number" ? record["confidence"] : undefined,
|
||||||
typeof record["confidence"] === "number"
|
expectedOutput: typeof record["expectedOutput"] === "string" ? record["expectedOutput"] : undefined,
|
||||||
? record["confidence"]
|
|
||||||
: undefined,
|
|
||||||
expectedOutput:
|
|
||||||
typeof record["expectedOutput"] === "string"
|
|
||||||
? record["expectedOutput"]
|
|
||||||
: undefined,
|
|
||||||
from,
|
from,
|
||||||
reason,
|
reason,
|
||||||
risk:
|
risk:
|
||||||
@@ -71,198 +73,26 @@ function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
|
|||||||
record["risk"] === "high"
|
record["risk"] === "high"
|
||||||
? record["risk"]
|
? record["risk"]
|
||||||
: undefined,
|
: undefined,
|
||||||
taskSpec:
|
taskSpec: typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
|
||||||
typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
|
|
||||||
to,
|
to,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function isHandoffReason(value: unknown): value is HandoffReason {
|
function asStringArray(value: unknown): string[] {
|
||||||
return (
|
return Array.isArray(value)
|
||||||
value === "out-of-scope" ||
|
? value.filter((item): item is string => typeof item === "string")
|
||||||
value === "missing-tool" ||
|
: [];
|
||||||
value === "low-confidence" ||
|
|
||||||
value === "validation-failed" ||
|
|
||||||
value === "same-agent-revisit" ||
|
|
||||||
value === "max-hop" ||
|
|
||||||
value === "high-risk-action" ||
|
|
||||||
value === "policy-risk"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function readPositiveNumber(
|
|
||||||
context: Record<string, unknown> | undefined,
|
|
||||||
key: string,
|
|
||||||
fallback: number,
|
|
||||||
): number {
|
|
||||||
const value = context?.[key];
|
|
||||||
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
||||||
? value
|
|
||||||
: fallback;
|
|
||||||
}
|
|
||||||
|
|
||||||
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
|
||||||
if (verdict === "same-agent-loop") {
|
|
||||||
return "same-agent-revisit";
|
|
||||||
}
|
|
||||||
if (verdict === "max-hops") {
|
|
||||||
return "max-hop";
|
|
||||||
}
|
|
||||||
return "policy-risk";
|
|
||||||
}
|
|
||||||
|
|
||||||
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
|
||||||
const value = response.metadata?.["handoffRequest"];
|
|
||||||
if (typeof value !== "object" || value === null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const request = value as Partial<HandoffRequest>;
|
|
||||||
if (
|
|
||||||
typeof request.targetAgent !== "string" ||
|
|
||||||
typeof request.reason !== "string" ||
|
|
||||||
typeof request.taskSpec !== "string"
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
confidence:
|
|
||||||
typeof request.confidence === "number" ? request.confidence : response.confidence,
|
|
||||||
evidenceRefs: Array.isArray(request.evidenceRefs)
|
|
||||||
? request.evidenceRefs.filter((item): item is string => typeof item === "string")
|
|
||||||
: [],
|
|
||||||
expectedOutput:
|
|
||||||
typeof request.expectedOutput === "string"
|
|
||||||
? request.expectedOutput
|
|
||||||
: "text",
|
|
||||||
reason: request.reason,
|
|
||||||
risk:
|
|
||||||
request.risk === "low" || request.risk === "medium" || request.risk === "high"
|
|
||||||
? request.risk
|
|
||||||
: "medium",
|
|
||||||
targetAgent: request.targetAgent,
|
|
||||||
taskSpec: request.taskSpec,
|
|
||||||
...(request.userVisibleMessage
|
|
||||||
? { userVisibleMessage: request.userVisibleMessage }
|
|
||||||
: {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeControlTokens(
|
|
||||||
response: AgentResponse,
|
|
||||||
currentSkill: string,
|
|
||||||
): AgentResponse {
|
|
||||||
const parsed = parseControlTokens(response.text);
|
|
||||||
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
|
||||||
if (handoff?.kind !== "handoff") {
|
|
||||||
return {
|
|
||||||
...response,
|
|
||||||
text: parsed.stripped,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
...response,
|
|
||||||
metadata: {
|
|
||||||
...(response.metadata ?? {}),
|
|
||||||
handoffRequest: {
|
|
||||||
confidence: response.confidence,
|
|
||||||
evidenceRefs: [],
|
|
||||||
expectedOutput: "text",
|
|
||||||
reason: handoff.reason,
|
|
||||||
risk: "medium",
|
|
||||||
targetAgent: handoff.targetAgent,
|
|
||||||
taskSpec: `Continue handling from ${currentSkill}`,
|
|
||||||
} satisfies HandoffRequest,
|
|
||||||
},
|
|
||||||
text: parsed.stripped,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function withHandoffMetadata(
|
|
||||||
response: AgentResponse,
|
|
||||||
chain: string[],
|
|
||||||
depth: number,
|
|
||||||
lastHandoff: HandoffRequest | undefined,
|
|
||||||
): AgentResponse {
|
|
||||||
return {
|
|
||||||
...response,
|
|
||||||
metadata: {
|
|
||||||
...(response.metadata ?? {}),
|
|
||||||
handoffChain: chain,
|
|
||||||
handoffDepth: depth,
|
|
||||||
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function plannedHandoffResponse(
|
|
||||||
request: AgentRequest,
|
|
||||||
step: EvalHandoffPlanStep,
|
|
||||||
): AgentResponse {
|
|
||||||
return {
|
|
||||||
confidence: step.confidence ?? 0.72,
|
|
||||||
metadata: {
|
|
||||||
handoffRequest: {
|
|
||||||
confidence: step.confidence ?? 0.72,
|
|
||||||
evidenceRefs: [],
|
|
||||||
expectedOutput: step.expectedOutput ?? "text",
|
|
||||||
reason: step.reason,
|
|
||||||
risk: step.risk ?? "medium",
|
|
||||||
targetAgent: step.to,
|
|
||||||
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
|
|
||||||
} satisfies HandoffRequest,
|
|
||||||
},
|
|
||||||
requestId: request.requestId,
|
|
||||||
source: "template",
|
|
||||||
text: `handoff ${step.from} to ${step.to}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function runActualSkill(
|
|
||||||
request: AgentRequest,
|
|
||||||
history: ConversationMessage[],
|
|
||||||
skill: Skill,
|
|
||||||
budget: BudgetTracker,
|
|
||||||
): Promise<AgentResponse> {
|
|
||||||
const response = normalizeControlTokens(
|
|
||||||
await skill.handle(request, history),
|
|
||||||
skill.name,
|
|
||||||
);
|
|
||||||
if (!response.text.trim() || readHandoffRequest(response)) {
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
const decision = decideEscalation({
|
|
||||||
budget: budget.snapshot(),
|
|
||||||
capabilityCard: skill.capabilityCard?.(),
|
|
||||||
classified: { confidence: response.confidence },
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
taskSpec: `Review ${skill.name} response for ${request.trigger}.`,
|
|
||||||
});
|
|
||||||
if (!decision.handoff) {
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
...response,
|
|
||||||
metadata: {
|
|
||||||
...(response.metadata ?? {}),
|
|
||||||
handoffRequest: decision.handoff,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runEvalDispatch(input: {
|
export async function runEvalDispatch(input: {
|
||||||
history?: ConversationMessage[] | undefined;
|
history?: ConversationMessage[] | undefined;
|
||||||
request: AgentRequest;
|
request: AgentRequest;
|
||||||
skills: readonly Skill[];
|
skills: readonly Skill[];
|
||||||
|
transitionMatrix?: TransitionMatrix | undefined;
|
||||||
}): Promise<EvalDispatchResult> {
|
}): Promise<EvalDispatchResult> {
|
||||||
const context = input.request.context;
|
const context = input.request.context;
|
||||||
const plan = readPlan(input.request);
|
|
||||||
const useActualSkills = asRecord(context)["evalUseActualSkills"] === true;
|
|
||||||
const budget = new BudgetTracker({
|
const budget = new BudgetTracker({
|
||||||
enabled: true,
|
enabled: true,
|
||||||
maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5),
|
maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5),
|
||||||
@@ -274,139 +104,40 @@ export async function runEvalDispatch(input: {
|
|||||||
),
|
),
|
||||||
maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20),
|
maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20),
|
||||||
});
|
});
|
||||||
|
const skill = selectSkillForRequest(input.skills, input.request);
|
||||||
let request = input.request;
|
|
||||||
let skill = selectSkillForRequest(input.skills, request);
|
|
||||||
if (!skill) {
|
if (!skill) {
|
||||||
throw new Error(`No skill can handle eval request: ${request.skill}`);
|
throw new Error(`No skill can handle eval request: ${input.request.skill}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
let chain = [skill.name];
|
|
||||||
let lastHandoff: HandoffRequest | undefined;
|
|
||||||
let planIndex = 0;
|
|
||||||
budget.recordVisit(skill.name);
|
budget.recordVisit(skill.name);
|
||||||
|
|
||||||
for (let depth = 0; depth <= 8; depth++) {
|
const plan: EvalDispatchPlan = {
|
||||||
const planned = plan[planIndex]?.from === skill.name ? plan[planIndex] : undefined;
|
maxDepth: readPositiveNumber(context, "evalMaxDispatchDepth", 8),
|
||||||
const response = planned
|
steps: readPlan(input.request),
|
||||||
? plannedHandoffResponse(request, planned)
|
useActualSkills: asRecord(context)["evalUseActualSkills"] === true,
|
||||||
: useActualSkills
|
|
||||||
? await runActualSkill(request, input.history ?? [], skill, budget)
|
|
||||||
: {
|
|
||||||
confidence: 1,
|
|
||||||
requestId: request.requestId,
|
|
||||||
source: "template" as const,
|
|
||||||
text: `handoff eval final: ${skill.name}`,
|
|
||||||
};
|
|
||||||
if (planned) {
|
|
||||||
planIndex += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const handoff = readHandoffRequest(response);
|
|
||||||
if (!handoff) {
|
|
||||||
const finalResponse = withHandoffMetadata(
|
|
||||||
response,
|
|
||||||
chain,
|
|
||||||
chain.length - 1,
|
|
||||||
lastHandoff,
|
|
||||||
);
|
|
||||||
return {
|
|
||||||
chain,
|
|
||||||
finalSkill: skill.name,
|
|
||||||
response: finalResponse,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const verdict = budget.assertBeforeHandoff(handoff.targetAgent);
|
|
||||||
const resolved: HandoffRequest =
|
|
||||||
verdict === "ok"
|
|
||||||
? handoff
|
|
||||||
: {
|
|
||||||
...handoff,
|
|
||||||
reason: budgetVerdictToReason(verdict),
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
};
|
|
||||||
const nextSkill = selectSkillForRequest(input.skills, {
|
|
||||||
...request,
|
|
||||||
skill: resolved.targetAgent,
|
|
||||||
trigger: "handoff",
|
|
||||||
});
|
|
||||||
if (!nextSkill) {
|
|
||||||
const fallbackChain = [...chain, "supervisor"];
|
|
||||||
const fallbackHandoff = {
|
|
||||||
...resolved,
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
};
|
|
||||||
return {
|
|
||||||
chain: fallbackChain,
|
|
||||||
finalSkill: "supervisor",
|
|
||||||
response: withHandoffMetadata(
|
|
||||||
{
|
|
||||||
confidence: 0.4,
|
|
||||||
requestId: request.requestId,
|
|
||||||
source: "template",
|
|
||||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
|
||||||
},
|
|
||||||
fallbackChain,
|
|
||||||
fallbackChain.length - 1,
|
|
||||||
fallbackHandoff,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
budget.recordHandoff(nextSkill.name);
|
|
||||||
const nextChain = [...chain, nextSkill.name];
|
|
||||||
lastHandoff = {
|
|
||||||
...resolved,
|
|
||||||
targetAgent: nextSkill.name,
|
|
||||||
};
|
|
||||||
const packet = buildHandoffPacket({
|
|
||||||
handoff: lastHandoff,
|
|
||||||
history: input.history,
|
|
||||||
request,
|
|
||||||
response,
|
|
||||||
sourceChain: nextChain,
|
|
||||||
state: {
|
|
||||||
source: skill.name,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
request = {
|
|
||||||
...request,
|
|
||||||
context: {
|
|
||||||
...(request.context ?? {}),
|
|
||||||
handoffPacket: packet,
|
|
||||||
},
|
|
||||||
skill: nextSkill.name,
|
|
||||||
trigger: "handoff",
|
|
||||||
};
|
|
||||||
skill = nextSkill;
|
|
||||||
chain = nextChain;
|
|
||||||
}
|
|
||||||
|
|
||||||
const finalHandoff: HandoffRequest = {
|
|
||||||
confidence: 0.4,
|
|
||||||
evidenceRefs: [],
|
|
||||||
expectedOutput: "text",
|
|
||||||
reason: "max-hop",
|
|
||||||
risk: "medium",
|
|
||||||
targetAgent: "supervisor",
|
|
||||||
taskSpec: "Eval dispatch exceeded max loop depth.",
|
|
||||||
};
|
};
|
||||||
const fallbackChain = chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"];
|
const response = await dispatch(
|
||||||
|
{
|
||||||
|
budget,
|
||||||
|
chain: [skill.name],
|
||||||
|
depth: 0,
|
||||||
|
history: input.history ?? [],
|
||||||
|
request: input.request,
|
||||||
|
skill,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
evalPlan: plan,
|
||||||
|
getSkillJudgeConfig,
|
||||||
|
selectSkillForRequest,
|
||||||
|
shouldApplyRuntimeVerifier: () => false,
|
||||||
|
skills: input.skills,
|
||||||
|
transitionMatrix: input.transitionMatrix,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const chain = asStringArray(response.metadata?.["handoffChain"]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
chain: fallbackChain,
|
chain,
|
||||||
finalSkill: "supervisor",
|
finalSkill: chain.at(-1) ?? skill.name,
|
||||||
response: withHandoffMetadata(
|
response,
|
||||||
{
|
|
||||||
confidence: 0.4,
|
|
||||||
requestId: input.request.requestId,
|
|
||||||
source: "template",
|
|
||||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
|
||||||
},
|
|
||||||
fallbackChain,
|
|
||||||
fallbackChain.length - 1,
|
|
||||||
finalHandoff,
|
|
||||||
),
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import type { StreamEntry } from "@common/garnet";
|
||||||
|
import { BACKGROUND_STREAM_KEY } from "./shadow-bus.js";
|
||||||
|
import { loadSupervisorOutcomes } from "./router-training-signals.js";
|
||||||
|
import {
|
||||||
|
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||||
|
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||||
|
} from "@skills/supervisor/review-queue";
|
||||||
|
|
||||||
|
class FakeGarnet {
|
||||||
|
constructor(private readonly streams: Record<string, StreamEntry[]>) {}
|
||||||
|
|
||||||
|
readStreamRange(key: string): Promise<StreamEntry[]> {
|
||||||
|
return Promise.resolve(this.streams[key] ?? []);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("loadSupervisorOutcomes", () => {
|
||||||
|
it("joins supervisor review entries, resolution audit, and shadow eval by requestId", async () => {
|
||||||
|
const garnet = new FakeGarnet({
|
||||||
|
[BACKGROUND_STREAM_KEY]: [
|
||||||
|
{
|
||||||
|
id: "10-0",
|
||||||
|
message: {
|
||||||
|
classified: "{}",
|
||||||
|
createdAt: "2026-05-18T00:00:30.000Z",
|
||||||
|
derived: "{}",
|
||||||
|
input: JSON.stringify({
|
||||||
|
request: {
|
||||||
|
chatroomId: "room-1",
|
||||||
|
nickname: "tester",
|
||||||
|
requestId: "request-1",
|
||||||
|
skill: "onboarding",
|
||||||
|
trigger: "user-message",
|
||||||
|
userId: "user-1",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
output: JSON.stringify({
|
||||||
|
confidence: 1,
|
||||||
|
requestId: "request-1",
|
||||||
|
source: "template",
|
||||||
|
text: "ok",
|
||||||
|
}),
|
||||||
|
skill: "onboarding",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[SUPERVISOR_RESOLUTIONS_STREAM_KEY]: [
|
||||||
|
{
|
||||||
|
id: "2-0",
|
||||||
|
message: {
|
||||||
|
action: "manual-reply",
|
||||||
|
resolvedAt: "2026-05-18T00:02:00.000Z",
|
||||||
|
resolvedBy: "operator@example.com",
|
||||||
|
reviewId: "1-0",
|
||||||
|
type: "resolution",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[SUPERVISOR_REVIEW_STREAM_KEY]: [
|
||||||
|
{
|
||||||
|
id: "1-0",
|
||||||
|
message: {
|
||||||
|
chatroomId: "room-1",
|
||||||
|
packet: JSON.stringify({
|
||||||
|
constraints: [],
|
||||||
|
currentState: {},
|
||||||
|
evidence: [],
|
||||||
|
expectedOutputSchema: "text",
|
||||||
|
handoffReason: "low-confidence",
|
||||||
|
sourceChain: ["onboarding", "supervisor"],
|
||||||
|
summary: "unclear",
|
||||||
|
userGoal: "help",
|
||||||
|
}),
|
||||||
|
queuedAt: "2026-05-18T00:01:00.000Z",
|
||||||
|
reason: "low-confidence",
|
||||||
|
requestId: "request-1",
|
||||||
|
target: "human-review",
|
||||||
|
userId: "user-1",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
loadSupervisorOutcomes(
|
||||||
|
garnet,
|
||||||
|
new Date("2026-05-18T00:00:00.000Z"),
|
||||||
|
new Date("2026-05-18T00:03:00.000Z"),
|
||||||
|
),
|
||||||
|
).resolves.toMatchObject([
|
||||||
|
{
|
||||||
|
reason: "low-confidence",
|
||||||
|
resolution: {
|
||||||
|
action: "manual-reply",
|
||||||
|
},
|
||||||
|
reviewId: "1-0",
|
||||||
|
shadowEvalEntry: {
|
||||||
|
request: {
|
||||||
|
requestId: "request-1",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sourceChain: ["onboarding", "supervisor"],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
import type { HandoffReason } from "@agent/types";
|
||||||
|
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||||
|
import {
|
||||||
|
parseSupervisorReviewEntry,
|
||||||
|
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||||
|
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||||
|
type SupervisorReviewAction,
|
||||||
|
type SupervisorReviewResolution,
|
||||||
|
} from "@skills/supervisor/review-queue";
|
||||||
|
import { BACKGROUND_STREAM_KEY, type ShadowEvalEnvelope } from "./shadow-bus.js";
|
||||||
|
|
||||||
|
export interface SupervisorOutcome {
|
||||||
|
queuedAt: string;
|
||||||
|
reason: HandoffReason;
|
||||||
|
resolution: {
|
||||||
|
action: SupervisorReviewAction;
|
||||||
|
resolvedAt: string;
|
||||||
|
};
|
||||||
|
reviewId: string;
|
||||||
|
shadowEvalEntry?: ShadowEvalEnvelope | undefined;
|
||||||
|
sourceChain: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseJson<T>(value: string | undefined): T | undefined {
|
||||||
|
if (!value?.trim()) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return JSON.parse(value) as T;
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWithinIso(
|
||||||
|
value: string,
|
||||||
|
since: Date,
|
||||||
|
until: Date,
|
||||||
|
): boolean {
|
||||||
|
const time = Date.parse(value);
|
||||||
|
return (
|
||||||
|
Number.isFinite(time) &&
|
||||||
|
time >= since.getTime() &&
|
||||||
|
time <= until.getTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseShadowEntry(
|
||||||
|
entry: StreamEntry,
|
||||||
|
): ShadowEvalEnvelope | null {
|
||||||
|
const input = parseJson<{ request?: ShadowEvalEnvelope["request"] }>(
|
||||||
|
entry.message["input"],
|
||||||
|
);
|
||||||
|
const response = parseJson<ShadowEvalEnvelope["response"]>(
|
||||||
|
entry.message["output"],
|
||||||
|
);
|
||||||
|
const skill = entry.message["skill"];
|
||||||
|
const createdAt = entry.message["createdAt"];
|
||||||
|
if (!input?.request || !response || !skill || !createdAt) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
classified: parseJson(entry.message["classified"]) ?? {},
|
||||||
|
createdAt,
|
||||||
|
derived: parseJson(entry.message["derived"]) ?? {},
|
||||||
|
request: input.request,
|
||||||
|
response,
|
||||||
|
skill,
|
||||||
|
...(entry.message["latencyMs"]
|
||||||
|
? { latencyMs: Number(entry.message["latencyMs"]) }
|
||||||
|
: {}),
|
||||||
|
...(entry.message["process"]
|
||||||
|
? { process: parseJson(entry.message["process"]) }
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSupervisorReviewAction(
|
||||||
|
value: string | undefined,
|
||||||
|
): value is SupervisorReviewAction {
|
||||||
|
return value === "manual-reply" || value === "reroute" || value === "dismiss";
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseResolutionEntry(
|
||||||
|
entry: StreamEntry,
|
||||||
|
): { resolution: SupervisorReviewResolution; reviewId: string } | null {
|
||||||
|
const reviewId = entry.message["reviewId"];
|
||||||
|
const resolvedAt = entry.message["resolvedAt"];
|
||||||
|
const resolvedBy = entry.message["resolvedBy"];
|
||||||
|
const action = entry.message["action"];
|
||||||
|
if (
|
||||||
|
!reviewId ||
|
||||||
|
!resolvedAt ||
|
||||||
|
!resolvedBy ||
|
||||||
|
!isSupervisorReviewAction(action)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
resolution: {
|
||||||
|
action,
|
||||||
|
resolvedAt,
|
||||||
|
resolvedBy,
|
||||||
|
...(entry.message["note"] ? { note: entry.message["note"] } : {}),
|
||||||
|
},
|
||||||
|
reviewId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function latestResolutionsByReviewId(
|
||||||
|
entries: StreamEntry[],
|
||||||
|
): Map<string, SupervisorReviewResolution> {
|
||||||
|
const byReviewId = new Map<string, SupervisorReviewResolution>();
|
||||||
|
for (const entry of entries) {
|
||||||
|
const parsed = parseResolutionEntry(entry);
|
||||||
|
if (!parsed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const current = byReviewId.get(parsed.reviewId);
|
||||||
|
if (!current || parsed.resolution.resolvedAt > current.resolvedAt) {
|
||||||
|
byReviewId.set(parsed.reviewId, parsed.resolution);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return byReviewId;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadSupervisorOutcomes(
|
||||||
|
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||||
|
since: Date,
|
||||||
|
until: Date,
|
||||||
|
): Promise<SupervisorOutcome[]> {
|
||||||
|
const [reviewEntries, resolutionEntries, shadowEntries] = await Promise.all([
|
||||||
|
garnetClient.readStreamRange(SUPERVISOR_REVIEW_STREAM_KEY, "-", "+"),
|
||||||
|
garnetClient.readStreamRange(SUPERVISOR_RESOLUTIONS_STREAM_KEY, "-", "+"),
|
||||||
|
garnetClient.readStreamRange(BACKGROUND_STREAM_KEY, "-", "+"),
|
||||||
|
]);
|
||||||
|
const shadowByRequestId = new Map(
|
||||||
|
shadowEntries.flatMap((entry) => {
|
||||||
|
const shadow = parseShadowEntry(entry);
|
||||||
|
return shadow ? [[shadow.request.requestId, shadow] as const] : [];
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const resolutionByReviewId = latestResolutionsByReviewId(resolutionEntries);
|
||||||
|
const outcomes: SupervisorOutcome[] = [];
|
||||||
|
|
||||||
|
for (const entry of reviewEntries) {
|
||||||
|
const review = parseSupervisorReviewEntry(entry);
|
||||||
|
if (!review || !isWithinIso(review.queuedAt, since, until)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const resolution = resolutionByReviewId.get(review.id);
|
||||||
|
if (!resolution || !isWithinIso(resolution.resolvedAt, since, until)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
outcomes.push({
|
||||||
|
queuedAt: review.queuedAt,
|
||||||
|
reason: review.reason,
|
||||||
|
resolution: {
|
||||||
|
action: resolution.action,
|
||||||
|
resolvedAt: resolution.resolvedAt,
|
||||||
|
},
|
||||||
|
reviewId: review.id,
|
||||||
|
shadowEvalEntry: shadowByRequestId.get(review.requestId),
|
||||||
|
sourceChain: review.packet?.sourceChain ?? [],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return outcomes;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const ROUTER_TRAINING_SIGNAL_STREAMS = {
|
||||||
|
background: BACKGROUND_STREAM_KEY,
|
||||||
|
resolutions: SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||||
|
reviews: SUPERVISOR_REVIEW_STREAM_KEY,
|
||||||
|
} as const;
|
||||||
@@ -6,7 +6,7 @@ import {
|
|||||||
getShadowEvalSamplingRate,
|
getShadowEvalSamplingRate,
|
||||||
} from "../config.js";
|
} from "../config.js";
|
||||||
|
|
||||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
export const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||||
|
|
||||||
export interface ShadowEvalEnvelope {
|
export interface ShadowEvalEnvelope {
|
||||||
classified: unknown;
|
classified: unknown;
|
||||||
|
|||||||
@@ -262,6 +262,7 @@ async function bootstrap(): Promise<void> {
|
|||||||
natsClient,
|
natsClient,
|
||||||
roomContextService,
|
roomContextService,
|
||||||
shadowEvalBus,
|
shadowEvalBus,
|
||||||
|
skillRegistry,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
+34
-24
@@ -25,31 +25,41 @@ describe("operations review CLI formatters", () => {
|
|||||||
|
|
||||||
it("formats packet details for show output", () => {
|
it("formats packet details for show output", () => {
|
||||||
expect(
|
expect(
|
||||||
formatReviewShow({
|
formatReviewShow(
|
||||||
chatroomId: "room-1",
|
{
|
||||||
id: "1-0",
|
chatroomId: "room-1",
|
||||||
packet: {
|
id: "1-0",
|
||||||
constraints: [],
|
packet: {
|
||||||
currentState: {},
|
constraints: [],
|
||||||
evidence: [
|
currentState: {},
|
||||||
{
|
evidence: [
|
||||||
id: "e1",
|
{
|
||||||
kind: "message",
|
id: "e1",
|
||||||
summary: "unclear message",
|
kind: "message",
|
||||||
},
|
summary: "unclear message",
|
||||||
],
|
},
|
||||||
expectedOutputSchema: "text",
|
],
|
||||||
handoffReason: "low-confidence",
|
expectedOutputSchema: "text",
|
||||||
sourceChain: ["onboarding", "supervisor"],
|
handoffReason: "low-confidence",
|
||||||
summary: "needs review",
|
sourceChain: ["onboarding", "supervisor"],
|
||||||
userGoal: "의도 확인",
|
summary: "needs review",
|
||||||
|
userGoal: "의도 확인",
|
||||||
|
},
|
||||||
|
queuedAt: "2026-05-18T00:00:00.000Z",
|
||||||
|
reason: "low-confidence",
|
||||||
|
requestId: "request-1",
|
||||||
|
target: "human-review",
|
||||||
|
userId: "user-1",
|
||||||
},
|
},
|
||||||
queuedAt: "2026-05-18T00:00:00.000Z",
|
[
|
||||||
reason: "low-confidence",
|
{
|
||||||
requestId: "request-1",
|
action: "manual-reply",
|
||||||
target: "human-review",
|
note: "handled",
|
||||||
userId: "user-1",
|
resolvedAt: "2026-05-18T01:00:00.000Z",
|
||||||
}),
|
resolvedBy: "operator@example.com",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
),
|
||||||
).toContain("sourceChain: onboarding > supervisor");
|
).toContain("sourceChain: onboarding > supervisor");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+23
-3
@@ -5,12 +5,14 @@ import { loadEnvironmentFromNodeEnv } from "@common/env-loader";
|
|||||||
import { connectGarnetForCli } from "@common/garnet-cli";
|
import { connectGarnetForCli } from "@common/garnet-cli";
|
||||||
import { initLogger } from "@obs/logger";
|
import { initLogger } from "@obs/logger";
|
||||||
import {
|
import {
|
||||||
|
getResolutionHistory,
|
||||||
getSupervisorReview,
|
getSupervisorReview,
|
||||||
getSupervisorReviewStats,
|
getSupervisorReviewStats,
|
||||||
listSupervisorReview,
|
listSupervisorReview,
|
||||||
resolveSupervisorReview,
|
resolveSupervisorReview,
|
||||||
type SupervisorReviewAction,
|
type SupervisorReviewAction,
|
||||||
type SupervisorReviewItem,
|
type SupervisorReviewItem,
|
||||||
|
type SupervisorReviewResolution,
|
||||||
type SupervisorReviewStats,
|
type SupervisorReviewStats,
|
||||||
} from "@skills/supervisor/review-queue";
|
} from "@skills/supervisor/review-queue";
|
||||||
import {
|
import {
|
||||||
@@ -95,12 +97,24 @@ export function formatReviewList(items: SupervisorReviewItem[]): string {
|
|||||||
return rows.map((row) => row.join("\t")).join("\n");
|
return rows.map((row) => row.join("\t")).join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
export function formatReviewShow(item: SupervisorReviewItem): string {
|
export function formatReviewShow(
|
||||||
|
item: SupervisorReviewItem,
|
||||||
|
history?: SupervisorReviewResolution[],
|
||||||
|
): string {
|
||||||
const packet = item.packet;
|
const packet = item.packet;
|
||||||
const evidence =
|
const evidence =
|
||||||
packet?.evidence
|
packet?.evidence
|
||||||
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
|
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
|
||||||
.join("\n") || "-";
|
.join("\n") || "-";
|
||||||
|
const historyLines =
|
||||||
|
history?.map((resolution) =>
|
||||||
|
[
|
||||||
|
`${resolution.resolvedAt} ${resolution.action} by ${resolution.resolvedBy}`,
|
||||||
|
resolution.note ? `note=${resolution.note}` : null,
|
||||||
|
]
|
||||||
|
.filter((value): value is string => Boolean(value))
|
||||||
|
.join(" "),
|
||||||
|
) ?? [];
|
||||||
|
|
||||||
return [
|
return [
|
||||||
`id: ${item.id}`,
|
`id: ${item.id}`,
|
||||||
@@ -124,6 +138,9 @@ export function formatReviewShow(item: SupervisorReviewItem): string {
|
|||||||
`note: ${stringifyOptional(item.resolution.note)}`,
|
`note: ${stringifyOptional(item.resolution.note)}`,
|
||||||
].join("\n")
|
].join("\n")
|
||||||
: "[resolution]\n-",
|
: "[resolution]\n-",
|
||||||
|
...(history
|
||||||
|
? ["[history]", ...(historyLines.length > 0 ? historyLines : ["-"])]
|
||||||
|
: []),
|
||||||
].join("\n");
|
].join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -191,10 +208,13 @@ async function handleReviewShow(
|
|||||||
console.error(`[review] entry not found: ${id}`);
|
console.error(`[review] entry not found: ${id}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const history = hasFlag(flags, "history")
|
||||||
|
? await getResolutionHistory(connection.garnetClient, id)
|
||||||
|
: undefined;
|
||||||
console.info(
|
console.info(
|
||||||
hasFlag(flags, "json")
|
hasFlag(flags, "json")
|
||||||
? JSON.stringify(item, null, 2)
|
? JSON.stringify({ ...item, ...(history ? { history } : {}) }, null, 2)
|
||||||
: formatReviewShow(item),
|
: formatReviewShow(item, history),
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
await connection.close();
|
await connection.close();
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import type { ShadowEvalBus } from "@eval/shadow-bus";
|
|||||||
import type { LlmClient } from "@llm/llm-client";
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import type { RoomContextService } from "@room/room-context-service";
|
import type { RoomContextService } from "@room/room-context-service";
|
||||||
import type { Skill } from "./skill.js";
|
import type { Skill } from "./skill.js";
|
||||||
|
import type { SkillRegistry } from "./skill-loader.js";
|
||||||
|
|
||||||
export interface SkillDeps {
|
export interface SkillDeps {
|
||||||
garnetClient: GarnetClient;
|
garnetClient: GarnetClient;
|
||||||
@@ -11,6 +12,7 @@ export interface SkillDeps {
|
|||||||
natsClient: NatsClient;
|
natsClient: NatsClient;
|
||||||
roomContextService?: RoomContextService | undefined;
|
roomContextService?: RoomContextService | undefined;
|
||||||
shadowEvalBus: ShadowEvalBus;
|
shadowEvalBus: ShadowEvalBus;
|
||||||
|
skillRegistry?: SkillRegistry | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SkillFactory = (deps: SkillDeps) => Skill;
|
export type SkillFactory = (deps: SkillDeps) => Skill;
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
name: supervisor
|
name: supervisor
|
||||||
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor.
|
description: LLM 기반 root cause 라우팅 supervisor.
|
||||||
version: 1
|
version: 1
|
||||||
priority: 10
|
priority: 10
|
||||||
domain: meta
|
domain: meta
|
||||||
@@ -19,4 +19,4 @@ resources:
|
|||||||
trustTier: T4
|
trustTier: T4
|
||||||
---
|
---
|
||||||
|
|
||||||
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
LLM 기반 root cause 분석과 규칙 fallback을 함께 사용하는 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||||
|
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
|
||||||
|
|
||||||
|
function formatSkill(manifest: SkillManifest): string {
|
||||||
|
return [
|
||||||
|
`- ${manifest.name}: ${manifest.description}`,
|
||||||
|
`domain=${manifest.domain}`,
|
||||||
|
`canHandle=${manifest.canHandle.join(", ") || "*"}`,
|
||||||
|
`cannotHandle=${manifest.cannotHandle.join(", ") || "-"}`,
|
||||||
|
`risk=${manifest.riskTier}`,
|
||||||
|
].join(" | ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSupervisorSystemPrompt(
|
||||||
|
registry: SkillRegistry,
|
||||||
|
_packet?: HandoffPacket,
|
||||||
|
): string {
|
||||||
|
const skills = registry
|
||||||
|
.list()
|
||||||
|
.filter((manifest) => manifest.name !== "supervisor");
|
||||||
|
return [
|
||||||
|
"너는 multi-agent 시스템의 supervisor다.",
|
||||||
|
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
|
||||||
|
"",
|
||||||
|
"[사용 가능한 specialist]",
|
||||||
|
...skills.map(formatSkill),
|
||||||
|
"",
|
||||||
|
"[응답 형식]",
|
||||||
|
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단"}',
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSupervisorUserPrompt(
|
||||||
|
packet?: HandoffPacket,
|
||||||
|
): string {
|
||||||
|
if (!packet) {
|
||||||
|
return "사용자 메시지가 명확하지 않음. 어디로 보낼지 결정해줘.";
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
`[escalation reason]: ${packet.handoffReason}`,
|
||||||
|
`[user goal]: ${packet.userGoal}`,
|
||||||
|
`[summary]: ${packet.summary}`,
|
||||||
|
`[source chain]: ${packet.sourceChain.join(" -> ")}`,
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
@@ -5,5 +5,10 @@ import { SupervisorSkill } from "./skill.js";
|
|||||||
export { SupervisorSkill } from "./skill.js";
|
export { SupervisorSkill } from "./skill.js";
|
||||||
|
|
||||||
export function createSupervisorSkill(deps: SkillDeps): Skill {
|
export function createSupervisorSkill(deps: SkillDeps): Skill {
|
||||||
return new SupervisorSkill(deps.garnetClient, deps.natsClient);
|
return new SupervisorSkill(
|
||||||
|
deps.garnetClient,
|
||||||
|
deps.natsClient,
|
||||||
|
deps.llmClient,
|
||||||
|
deps.skillRegistry,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
@@ -4,15 +4,18 @@ import type { StreamEntry } from "@common/garnet";
|
|||||||
import {
|
import {
|
||||||
getSupervisorReview,
|
getSupervisorReview,
|
||||||
getSupervisorReviewStats,
|
getSupervisorReviewStats,
|
||||||
|
getResolutionHistory,
|
||||||
listSupervisorReview,
|
listSupervisorReview,
|
||||||
parseSupervisorReviewEntry,
|
parseSupervisorReviewEntry,
|
||||||
resolveSupervisorReview,
|
resolveSupervisorReview,
|
||||||
SUPERVISOR_RESOLVED_HASH_PREFIX,
|
SUPERVISOR_RESOLVED_HASH_PREFIX,
|
||||||
|
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||||
SUPERVISOR_REVIEW_STREAM_KEY,
|
SUPERVISOR_REVIEW_STREAM_KEY,
|
||||||
} from "./review-queue.js";
|
} from "./review-queue.js";
|
||||||
|
|
||||||
class FakeGarnet {
|
class FakeGarnet {
|
||||||
readonly hashes = new Map<string, Record<string, string>>();
|
readonly hashes = new Map<string, Record<string, string>>();
|
||||||
|
readonly resolutionStream: StreamEntry[] = [];
|
||||||
readonly stream: StreamEntry[];
|
readonly stream: StreamEntry[];
|
||||||
|
|
||||||
constructor(stream: StreamEntry[]) {
|
constructor(stream: StreamEntry[]) {
|
||||||
@@ -25,8 +28,12 @@ class FakeGarnet {
|
|||||||
end: string,
|
end: string,
|
||||||
options?: { count?: number | undefined },
|
options?: { count?: number | undefined },
|
||||||
): Promise<StreamEntry[]> {
|
): Promise<StreamEntry[]> {
|
||||||
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY);
|
expect([SUPERVISOR_REVIEW_STREAM_KEY, SUPERVISOR_RESOLUTIONS_STREAM_KEY]).toContain(key);
|
||||||
const entries = this.stream.filter((entry) => {
|
const source =
|
||||||
|
key === SUPERVISOR_RESOLUTIONS_STREAM_KEY
|
||||||
|
? this.resolutionStream
|
||||||
|
: this.stream;
|
||||||
|
const entries = source.filter((entry) => {
|
||||||
const afterStart = start === "-" || entry.id >= start;
|
const afterStart = start === "-" || entry.id >= start;
|
||||||
const beforeEnd = end === "+" || entry.id <= end;
|
const beforeEnd = end === "+" || entry.id <= end;
|
||||||
return afterStart && beforeEnd;
|
return afterStart && beforeEnd;
|
||||||
@@ -43,6 +50,16 @@ class FakeGarnet {
|
|||||||
return Promise.resolve();
|
return Promise.resolve();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
appendStream(key: string, fields: Record<string, string>): Promise<string> {
|
||||||
|
expect(key).toBe(SUPERVISOR_RESOLUTIONS_STREAM_KEY);
|
||||||
|
const id = `${this.resolutionStream.length + 1}-0`;
|
||||||
|
this.resolutionStream.push({
|
||||||
|
id,
|
||||||
|
message: { ...fields },
|
||||||
|
});
|
||||||
|
return Promise.resolve(id);
|
||||||
|
}
|
||||||
|
|
||||||
expireKey(): Promise<boolean> {
|
expireKey(): Promise<boolean> {
|
||||||
return Promise.resolve(true);
|
return Promise.resolve(true);
|
||||||
}
|
}
|
||||||
@@ -140,6 +157,35 @@ describe("supervisor review queue", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("dual-writes resolution updates to an audit stream", async () => {
|
||||||
|
const garnet = new FakeGarnet(entries);
|
||||||
|
await resolveSupervisorReview(garnet, "1-0", {
|
||||||
|
action: "manual-reply",
|
||||||
|
note: "handled",
|
||||||
|
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||||
|
resolvedBy: "operator@example.com",
|
||||||
|
});
|
||||||
|
await resolveSupervisorReview(garnet, "1-0", {
|
||||||
|
action: "dismiss",
|
||||||
|
resolvedAt: "2026-05-18T03:00:00.000Z",
|
||||||
|
resolvedBy: "operator-2@example.com",
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(getResolutionHistory(garnet, "1-0")).resolves.toEqual([
|
||||||
|
{
|
||||||
|
action: "dismiss",
|
||||||
|
resolvedAt: "2026-05-18T03:00:00.000Z",
|
||||||
|
resolvedBy: "operator-2@example.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
action: "manual-reply",
|
||||||
|
note: "handled",
|
||||||
|
resolvedAt: "2026-05-18T02:00:00.000Z",
|
||||||
|
resolvedBy: "operator@example.com",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("computes reason totals and top source chains", async () => {
|
it("computes reason totals and top source chains", async () => {
|
||||||
const garnet = new FakeGarnet(entries);
|
const garnet = new FakeGarnet(entries);
|
||||||
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import type { HandoffPacket } from "@agent/handoff-packet";
|
|||||||
import type { HandoffReason } from "@agent/types";
|
import type { HandoffReason } from "@agent/types";
|
||||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||||
import {
|
import {
|
||||||
|
getSupervisorResolutionsStreamMaxLen,
|
||||||
getSupervisorReviewQueueMaxLen,
|
getSupervisorReviewQueueMaxLen,
|
||||||
getSupervisorReviewResolutionTtlSec,
|
getSupervisorReviewResolutionTtlSec,
|
||||||
} from "../../config.js";
|
} from "../../config.js";
|
||||||
@@ -9,6 +10,8 @@ import {
|
|||||||
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
|
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
|
||||||
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
|
export const SUPERVISOR_RESOLVED_HASH_PREFIX =
|
||||||
"lolympus-agent:review-resolved:";
|
"lolympus-agent:review-resolved:";
|
||||||
|
export const SUPERVISOR_RESOLUTIONS_STREAM_KEY =
|
||||||
|
"lolympus-agent:review-resolutions";
|
||||||
|
|
||||||
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
|
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
|
||||||
|
|
||||||
@@ -67,7 +70,7 @@ type SupervisorReviewListGarnetClient = Pick<
|
|||||||
|
|
||||||
type SupervisorReviewResolveGarnetClient = Pick<
|
type SupervisorReviewResolveGarnetClient = Pick<
|
||||||
GarnetClient,
|
GarnetClient,
|
||||||
"expireKey" | "setHash"
|
"appendStream" | "expireKey" | "setHash"
|
||||||
>;
|
>;
|
||||||
|
|
||||||
const BASE_FIELDS = new Set([
|
const BASE_FIELDS = new Set([
|
||||||
@@ -285,6 +288,36 @@ export async function resolveSupervisorReview(
|
|||||||
const key = resolutionKey(id);
|
const key = resolutionKey(id);
|
||||||
await garnetClient.setHash(key, fields);
|
await garnetClient.setHash(key, fields);
|
||||||
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
|
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
|
||||||
|
await garnetClient.appendStream(
|
||||||
|
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||||
|
{
|
||||||
|
...fields,
|
||||||
|
reviewId: id,
|
||||||
|
type: "resolution",
|
||||||
|
},
|
||||||
|
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getResolutionHistory(
|
||||||
|
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||||
|
id: string,
|
||||||
|
): Promise<SupervisorReviewResolution[]> {
|
||||||
|
const entries = await garnetClient.readStreamRange(
|
||||||
|
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
|
||||||
|
"-",
|
||||||
|
"+",
|
||||||
|
);
|
||||||
|
|
||||||
|
return entries
|
||||||
|
.flatMap((entry) => {
|
||||||
|
if (entry.message["reviewId"] !== id) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const resolution = parseResolution(entry.message);
|
||||||
|
return resolution ? [resolution] : [];
|
||||||
|
})
|
||||||
|
.sort((left, right) => right.resolvedAt.localeCompare(left.resolvedAt));
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getSupervisorReviewStats(
|
export async function getSupervisorReviewStats(
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
export const supervisorRoutingSchema = z.object({
|
||||||
|
confidence: z.number().min(0).max(1),
|
||||||
|
reason: z.string().max(200),
|
||||||
|
target: z.string().min(1),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type SupervisorRoutingDecision = z.infer<typeof supervisorRoutingSchema>;
|
||||||
@@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest";
|
|||||||
import type { AlertPayload } from "@common/nats-subjects";
|
import type { AlertPayload } from "@common/nats-subjects";
|
||||||
import { buildHandoffPacket } from "@agent/handoff-packet";
|
import { buildHandoffPacket } from "@agent/handoff-packet";
|
||||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
|
import type { SkillRegistry } from "@skills/skill-loader";
|
||||||
import { SupervisorSkill } from "./skill.js";
|
import { SupervisorSkill } from "./skill.js";
|
||||||
|
|
||||||
class FakeGarnet {
|
class FakeGarnet {
|
||||||
@@ -27,6 +29,44 @@ class FakeNats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class FakeLlm {
|
||||||
|
chatCompletion(): Promise<{
|
||||||
|
content: string;
|
||||||
|
endpoint: string;
|
||||||
|
message: { content: string; role: "assistant" };
|
||||||
|
provider: "llama";
|
||||||
|
raw: object;
|
||||||
|
}> {
|
||||||
|
return Promise.resolve({
|
||||||
|
content: JSON.stringify({
|
||||||
|
confidence: 0.9,
|
||||||
|
reason: "광고 규칙 문의라 moderation이 적합",
|
||||||
|
target: "moderation",
|
||||||
|
}),
|
||||||
|
endpoint: "memory",
|
||||||
|
message: { content: "", role: "assistant" },
|
||||||
|
provider: "llama",
|
||||||
|
raw: {},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const registry = {
|
||||||
|
get: (name: string) => (name === "moderation" ? { name } : null),
|
||||||
|
list: () => [
|
||||||
|
{
|
||||||
|
canHandle: ["rules"],
|
||||||
|
cannotHandle: [],
|
||||||
|
description: "커뮤니티 규칙 처리",
|
||||||
|
domain: "community",
|
||||||
|
handoffTargets: [],
|
||||||
|
name: "moderation",
|
||||||
|
outputContract: {},
|
||||||
|
riskTier: "medium",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown as SkillRegistry;
|
||||||
|
|
||||||
const baseRequest: AgentRequest = {
|
const baseRequest: AgentRequest = {
|
||||||
chatroomId: "room-1",
|
chatroomId: "room-1",
|
||||||
nickname: "tester",
|
nickname: "tester",
|
||||||
@@ -70,6 +110,7 @@ describe("SupervisorSkill", () => {
|
|||||||
|
|
||||||
expect(response.text).toContain("운영자");
|
expect(response.text).toContain("운영자");
|
||||||
expect(response.metadata).toMatchObject({
|
expect(response.metadata).toMatchObject({
|
||||||
|
routedBy: "rules",
|
||||||
supervisorReason: "max-hop",
|
supervisorReason: "max-hop",
|
||||||
suggestedTarget: "onboarding",
|
suggestedTarget: "onboarding",
|
||||||
});
|
});
|
||||||
@@ -96,4 +137,44 @@ describe("SupervisorSkill", () => {
|
|||||||
metric: "agent.supervisor_handoff",
|
metric: "agent.supervisor_handoff",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses LLM routing when the decision is confident and valid", async () => {
|
||||||
|
const garnet = new FakeGarnet();
|
||||||
|
const nats = new FakeNats();
|
||||||
|
const skill = new SupervisorSkill(
|
||||||
|
garnet,
|
||||||
|
nats,
|
||||||
|
new FakeLlm() as unknown as Pick<LlmClient, "chatCompletion">,
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
const packet = buildHandoffPacket({
|
||||||
|
handoff: {
|
||||||
|
confidence: 0.7,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "out-of-scope",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "광고 규칙 문의",
|
||||||
|
},
|
||||||
|
request: baseRequest,
|
||||||
|
response: sourceResponse,
|
||||||
|
sourceChain: ["onboarding", "supervisor"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await skill.handle(
|
||||||
|
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.metadata).toMatchObject({
|
||||||
|
routedBy: "llm",
|
||||||
|
supervisorReason: "광고 규칙 문의라 moderation이 적합",
|
||||||
|
suggestedTarget: "moderation",
|
||||||
|
});
|
||||||
|
expect(garnet.streams[0]?.fields).toMatchObject({
|
||||||
|
reason: "out-of-scope",
|
||||||
|
target: "moderation",
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -4,14 +4,29 @@ import type {
|
|||||||
ConversationMessage,
|
ConversationMessage,
|
||||||
} from "@agent/types";
|
} from "@agent/types";
|
||||||
import { readHandoffPacket } from "@agent/handoff-packet";
|
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||||
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||||
import type { GarnetClient } from "@common/garnet";
|
import type { GarnetClient } from "@common/garnet";
|
||||||
import type { NatsClient } from "@common/nats";
|
import type { NatsClient } from "@common/nats";
|
||||||
|
import type { LlmClient } from "@llm/llm-client";
|
||||||
import { AlertService } from "@obs/alert";
|
import { AlertService } from "@obs/alert";
|
||||||
import { metrics } from "@obs/metrics";
|
import { metrics } from "@obs/metrics";
|
||||||
import type { Skill } from "@skills/skill";
|
import type { Skill } from "@skills/skill";
|
||||||
import type { SkillCapabilityCard } from "@skills/skill-loader";
|
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
||||||
|
import { getLogger } from "@obs/logger";
|
||||||
|
import {
|
||||||
|
getSupervisorLlmConfidenceThreshold,
|
||||||
|
getSupervisorLlmEnabled,
|
||||||
|
} from "../../config.js";
|
||||||
|
import {
|
||||||
|
buildSupervisorSystemPrompt,
|
||||||
|
buildSupervisorUserPrompt,
|
||||||
|
} from "./capability-prompt.js";
|
||||||
import { appendSupervisorReview } from "./review-queue.js";
|
import { appendSupervisorReview } from "./review-queue.js";
|
||||||
import { routeSupervisorTarget } from "./routing-rules.js";
|
import { routeSupervisorTarget } from "./routing-rules.js";
|
||||||
|
import {
|
||||||
|
supervisorRoutingSchema,
|
||||||
|
type SupervisorRoutingDecision,
|
||||||
|
} from "./schema.js";
|
||||||
|
|
||||||
const CAPABILITY_CARD: SkillCapabilityCard = {
|
const CAPABILITY_CARD: SkillCapabilityCard = {
|
||||||
canHandle: ["*"],
|
canHandle: ["*"],
|
||||||
@@ -34,6 +49,8 @@ export class SupervisorSkill implements Skill {
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
|
||||||
natsClient: Pick<NatsClient, "publish">,
|
natsClient: Pick<NatsClient, "publish">,
|
||||||
|
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||||
|
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||||
) {
|
) {
|
||||||
this.alertService = new AlertService(garnetClient, natsClient);
|
this.alertService = new AlertService(garnetClient, natsClient);
|
||||||
}
|
}
|
||||||
@@ -51,7 +68,11 @@ export class SupervisorSkill implements Skill {
|
|||||||
_conversationHistory: ConversationMessage[],
|
_conversationHistory: ConversationMessage[],
|
||||||
): Promise<AgentResponse> {
|
): Promise<AgentResponse> {
|
||||||
const packet = readHandoffPacket(request);
|
const packet = readHandoffPacket(request);
|
||||||
const target = routeSupervisorTarget(packet);
|
const llmDecision = await this.routeByLlm(packet);
|
||||||
|
const useLlmRoute =
|
||||||
|
llmDecision !== null &&
|
||||||
|
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||||
|
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
|
||||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||||
|
|
||||||
await appendSupervisorReview(this.garnetClient, {
|
await appendSupervisorReview(this.garnetClient, {
|
||||||
@@ -83,7 +104,8 @@ export class SupervisorSkill implements Skill {
|
|||||||
return {
|
return {
|
||||||
confidence: 0.6,
|
confidence: 0.6,
|
||||||
metadata: {
|
metadata: {
|
||||||
supervisorReason: reason,
|
routedBy: useLlmRoute ? "llm" : "rules",
|
||||||
|
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||||
suggestedTarget: target,
|
suggestedTarget: target,
|
||||||
},
|
},
|
||||||
requestId: request.requestId,
|
requestId: request.requestId,
|
||||||
@@ -91,4 +113,54 @@ export class SupervisorSkill implements Skill {
|
|||||||
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async routeByLlm(
|
||||||
|
packet?: HandoffPacket,
|
||||||
|
): Promise<SupervisorRoutingDecision | null> {
|
||||||
|
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await this.llmClient.chatCompletion(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
|
||||||
|
role: "system",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
content: buildSupervisorUserPrompt(packet),
|
||||||
|
role: "user",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{
|
||||||
|
jsonMode: true,
|
||||||
|
maxTokens: 150,
|
||||||
|
temperature: 0,
|
||||||
|
tier: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if (!result?.content.trim()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = supervisorRoutingSchema.safeParse(
|
||||||
|
JSON.parse(result.content) as unknown,
|
||||||
|
);
|
||||||
|
if (!parsed.success || !this.isKnownTarget(parsed.data.target)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return parsed.data;
|
||||||
|
} catch (error) {
|
||||||
|
getLogger().warn(
|
||||||
|
{ err: error },
|
||||||
|
"[Supervisor] LLM routing failed, falling back to rules",
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private isKnownTarget(target: string): boolean {
|
||||||
|
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user