Unify dispatch and add supervisor routing signals

This commit is contained in:
dahyeon committed 2026-05-18 18:42:35 -07:00
1 parent f0b00e5fd2
commit b63985694a
19 files changed
+1345 -812

No files matched your search

+37 -438
View File
@@ -3,22 +3,14 @@ import type { ConversationStore } from "@conv/conversation-store";
import type { NatsClient } from "@common/nats"; import type { NatsClient } from "@common/nats";
import { getRequestLogger, runWithContext } from "@obs/context"; import { getRequestLogger, runWithContext } from "@obs/context";
import { getLogger } from "@obs/logger"; import { getLogger } from "@obs/logger";
import { withSkillSpan, withSpan } from "@obs/spans"; import { withSpan } from "@obs/spans";
import type { Skill } from "@skills/skill"; import type { Skill } from "@skills/skill";
import type { TransitionMatrix } from "@skills/transition-matrix"; import type { TransitionMatrix } from "@skills/transition-matrix";
import { parseControlTokens } from "@skills/control-token-parser";
import { decide as decideEscalation } from "@skills/escalation/trigger";
import { getSkillJudgeConfig } from "@eval/judges/skill-config"; import { getSkillJudgeConfig } from "@eval/judges/skill-config";
import { metrics } from "@obs/metrics"; import { metrics } from "@obs/metrics";
import { BudgetTracker, type BudgetVerdict } from "./budget.js"; import { BudgetTracker } from "./budget.js";
import { buildHandoffPacket } from "./handoff-packet.js"; import { dispatch } from "./dispatch.js";
import type { import type { AgentRequest, AgentResponse } from "./types.js";
AgentRequest,
AgentResponse,
ConversationMessage,
HandoffReason,
HandoffRequest,
} from "./types.js";
import { import {
getAgentRuntimeVerifierEnabled, getAgentRuntimeVerifierEnabled,
getAgentRequestTimeoutMs, getAgentRequestTimeoutMs,
@@ -167,14 +159,39 @@ export class Agent {
const startedAt = Date.now(); const startedAt = Date.now();
const budget = new BudgetTracker(); const budget = new BudgetTracker();
budget.recordVisit(skill.name); budget.recordVisit(skill.name);
const response = await this.dispatch({ const response = await dispatch(
budget, {
chain: [skill.name], budget,
depth: 0, chain: [skill.name],
history, depth: 0,
request, history,
skill, request,
}); skill,
},
{
getSkillJudgeConfig,
onHandoff: (event) => {
metrics.handoffTotal.inc({
from: event.from,
reason: event.reason,
to: event.to,
});
},
onHandoffLoop: (event) => {
metrics.handoffLoopTotal.inc({ from: event.from });
},
onValidatorFail: (event) => {
metrics.validatorFailTotal.inc({
skill: event.skill,
stage: event.stage,
});
},
selectSkillForRequest,
shouldApplyRuntimeVerifier: getAgentRuntimeVerifierEnabled,
skills: this.skills,
transitionMatrix: this.transitionMatrix,
},
);
if (response.text.trim()) { if (response.text.trim()) {
await this.conversationStore.appendMessage( await this.conversationStore.appendMessage(
@@ -230,422 +247,4 @@ export class Agent {
}, },
); );
} }
private async dispatch(input: {
budget: BudgetTracker;
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}): Promise<AgentResponse> {
const { budget, chain, depth, history, request, skill } = input;
const contextBase = {
chatroomId: request.chatroomId,
deadlineAt: new Date(
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
).toISOString(),
requestId: request.requestId,
skill: skill.name,
startedAt: new Date().toISOString(),
trigger: request.trigger,
userId: request.userId,
};
return runWithContext(
{
...contextBase,
budget: budget.snapshot(),
handoffChain: chain,
handoffDepth: depth,
},
async () => {
const response = await this.runSkillOnce({
chain,
depth,
history,
request,
skill,
});
const normalized = this.normalizeControlTokens(response, skill.name);
const verified = await this.applyRuntimeVerifier({
chain,
depth,
history,
request,
response: normalized,
skill,
});
const escalated = this.applyEscalationTrigger({
budget,
request,
response: verified,
skill,
});
const annotated = this.withHandoffMetadata(escalated, chain, depth);
const handoff = readHandoffRequest(annotated);
if (!handoff) {
return annotated;
}
const next = this.prepareHandoff({
budget,
chain,
currentSkill: skill.name,
depth,
handoff,
history,
request,
response: annotated,
});
if (!next) {
return this.withHandoffMetadata(
this.makeSupervisorFallback(request, handoff.reason),
[...chain, "supervisor"],
depth + 1,
);
}
return this.dispatch(next);
},
);
}
private withHandoffMetadata(
response: AgentResponse,
chain: string[],
depth: number,
): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffChain: chain,
handoffDepth: depth,
},
};
}
private async runSkillOnce(input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}): Promise<AgentResponse> {
return withSkillSpan(
input.skill.name,
{
"agent.handoff_chain": input.chain.join(">"),
"agent.handoff_depth": input.depth,
"agent.skill": input.skill.name,
"agent.trigger": input.request.trigger,
},
() => input.skill.handle(input.request, input.history),
);
}
private normalizeControlTokens(
response: AgentResponse,
currentSkill: string,
): AgentResponse {
const parsed = parseControlTokens(response.text);
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
const metadata = { ...(response.metadata ?? {}) };
if (handoff?.kind === "handoff") {
metadata.handoffRequest = {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: handoff.reason,
risk: "medium",
targetAgent: handoff.targetAgent,
taskSpec: `Continue handling from ${currentSkill}`,
} satisfies HandoffRequest;
}
return {
...response,
metadata,
text: parsed.stripped,
};
}
private async applyRuntimeVerifier(input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
response: AgentResponse;
skill: Skill;
}): Promise<AgentResponse> {
if (!getAgentRuntimeVerifierEnabled()) {
return input.response;
}
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
if (!verifier) {
return input.response;
}
const verdict = await verifier({
request: input.request,
response: input.response,
skill: input.skill.name,
});
if (verdict === "pass") {
return input.response;
}
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
if (verdict === "retry" && input.depth === 0) {
const retryResponse = await this.runSkillOnce({
...input,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
runtimeVerifierRetry: true,
},
},
});
const normalizedRetry = this.normalizeControlTokens(
retryResponse,
input.skill.name,
);
if (readHandoffRequest(normalizedRetry)) {
return normalizedRetry;
}
const retryVerdict = await verifier({
request: input.request,
response: normalizedRetry,
skill: input.skill.name,
});
if (retryVerdict === "pass") {
return {
...normalizedRetry,
metadata: {
...(normalizedRetry.metadata ?? {}),
runtimeVerifier: "retry-pass",
},
};
}
metrics.validatorFailTotal.inc({
skill: input.skill.name,
stage: `retry-${retryVerdict}`,
});
return this.withRuntimeVerifierHandoff(normalizedRetry);
}
return this.withRuntimeVerifierHandoff(input.response);
}
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: "validation-failed",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "Runtime verifier rejected the response.",
} satisfies HandoffRequest,
},
};
}
private applyEscalationTrigger(input: {
budget: BudgetTracker;
request: AgentRequest;
response: AgentResponse;
skill: Skill;
}): AgentResponse {
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
return input.response;
}
const decision = decideEscalation({
budget: input.budget.snapshot(),
capabilityCard: input.skill.capabilityCard?.(),
classified: { confidence: input.response.confidence },
targetAgent: "supervisor",
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
});
if (!decision.handoff) {
return input.response;
}
return {
...input.response,
metadata: {
...(input.response.metadata ?? {}),
handoffRequest: decision.handoff,
},
};
}
private prepareHandoff(input: {
budget: BudgetTracker;
chain: string[];
currentSkill: string;
depth: number;
handoff: HandoffRequest;
history: ConversationMessage[];
request: AgentRequest;
response: AgentResponse;
}):
| {
budget: BudgetTracker;
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}
| null {
const target = this.resolveHandoffTarget(input);
const nextSkill = selectSkillForRequest(this.skills, {
...input.request,
skill: target.targetAgent,
trigger: "handoff",
});
if (!nextSkill) {
return null;
}
input.budget.recordHandoff(nextSkill.name);
metrics.handoffTotal.inc({
from: input.currentSkill,
reason: target.reason,
to: nextSkill.name,
});
const nextChain = [...input.chain, nextSkill.name];
const packet = buildHandoffPacket({
handoff: {
...input.handoff,
reason: target.reason,
targetAgent: nextSkill.name,
},
history: input.history,
request: input.request,
response: input.response,
sourceChain: nextChain,
state: {
source: input.currentSkill,
},
});
return {
budget: input.budget,
chain: nextChain,
depth: input.depth + 1,
history: input.history,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
handoffPacket: packet,
},
skill: nextSkill.name,
trigger: "handoff",
},
skill: nextSkill,
};
}
private resolveHandoffTarget(input: {
budget: BudgetTracker;
currentSkill: string;
handoff: HandoffRequest;
}): { reason: HandoffReason; targetAgent: string } {
const requestedTarget = input.handoff.targetAgent;
const allowed = this.transitionMatrix?.isAllowed(
input.currentSkill,
requestedTarget,
) ?? true;
if (!allowed) {
return {
reason: "policy-risk",
targetAgent: "supervisor",
};
}
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
if (verdict === "ok") {
return {
reason: input.handoff.reason,
targetAgent: requestedTarget,
};
}
if (verdict === "same-agent-loop") {
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
}
return {
reason: budgetVerdictToReason(verdict),
targetAgent: "supervisor",
};
}
private makeSupervisorFallback(
request: AgentRequest,
reason: HandoffReason,
): AgentResponse {
return {
confidence: 0.4,
metadata: {
supervisorFallbackReason: reason,
},
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
};
}
}
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
if (verdict === "same-agent-loop") {
return "same-agent-revisit";
}
if (verdict === "max-hops") {
return "max-hop";
}
return "policy-risk";
}
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
const value = response.metadata?.["handoffRequest"];
if (!value || typeof value !== "object") {
return null;
}
const handoff = value as Partial<HandoffRequest>;
if (
typeof handoff.targetAgent !== "string" ||
typeof handoff.reason !== "string" ||
typeof handoff.taskSpec !== "string" ||
typeof handoff.confidence !== "number" ||
typeof handoff.expectedOutput !== "string"
) {
return null;
}
return {
confidence: handoff.confidence,
evidenceRefs: Array.isArray(handoff.evidenceRefs)
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
: [],
expectedOutput: handoff.expectedOutput,
reason: handoff.reason,
risk:
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
? handoff.risk
: "medium",
targetAgent: handoff.targetAgent,
taskSpec: handoff.taskSpec,
...(handoff.userVisibleMessage
? { userVisibleMessage: handoff.userVisibleMessage }
: {}),
};
} }
+581
View File
@@ -0,0 +1,581 @@
import { getSkillJudgeConfig as defaultGetSkillJudgeConfig } from "@eval/judges/skill-config";
import type { SkillJudgeConfig } from "@eval/judges/skill-config";
import { runWithContext } from "@obs/context";
import { withSkillSpan } from "@obs/spans";
import { parseControlTokens } from "@skills/control-token-parser";
import { decide as decideEscalation } from "@skills/escalation/trigger";
import type { Skill } from "@skills/skill";
import type { TransitionMatrix } from "@skills/transition-matrix";
import { getAgentRequestTimeoutMs, getSafetyMarginMs } from "../config.js";
import type { BudgetTracker, BudgetVerdict } from "./budget.js";
import { buildHandoffPacket } from "./handoff-packet.js";
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
HandoffReason,
HandoffRequest,
} from "./types.js";
export interface HandoffEvent {
from: string;
reason: HandoffReason;
to: string;
}
export interface ValidatorFailEvent {
skill: string;
stage: string;
}
export interface EvalDispatchPlanStep {
confidence?: number | undefined;
expectedOutput?: string | undefined;
from: string;
reason: HandoffReason;
risk?: HandoffRequest["risk"] | undefined;
taskSpec?: string | undefined;
to: string;
}
export interface EvalDispatchPlan {
maxDepth?: number | undefined;
steps: EvalDispatchPlanStep[];
useActualSkills?: boolean | undefined;
}
export interface DispatchDependencies {
evalPlan?: EvalDispatchPlan | undefined;
getSkillJudgeConfig?: ((skillName: string) => SkillJudgeConfig) | undefined;
onHandoff?: ((event: HandoffEvent) => void) | undefined;
onHandoffLoop?: ((event: { from: string }) => void) | undefined;
onValidatorFail?: ((event: ValidatorFailEvent) => void) | undefined;
selectSkillForRequest: (
skills: readonly Skill[],
request: AgentRequest,
) => Skill | null;
shouldApplyRuntimeVerifier: () => boolean;
skills: readonly Skill[];
transitionMatrix?: TransitionMatrix | undefined;
}
export interface DispatchInput {
budget: BudgetTracker;
chain: string[];
depth: number;
evalPlanIndex?: number | undefined;
history: ConversationMessage[];
lastHandoff?: HandoffRequest | undefined;
request: AgentRequest;
skill: Skill;
}
export async function dispatch(
input: DispatchInput,
deps: DispatchDependencies,
): Promise<AgentResponse> {
const { budget, chain, depth, history, lastHandoff, request, skill } = input;
const contextBase = {
chatroomId: request.chatroomId,
deadlineAt: new Date(
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
).toISOString(),
requestId: request.requestId,
skill: skill.name,
startedAt: new Date().toISOString(),
trigger: request.trigger,
userId: request.userId,
};
return runWithContext(
{
...contextBase,
budget: budget.snapshot(),
handoffChain: chain,
handoffDepth: depth,
},
async () => {
const planStep = getCurrentPlanStep(input, deps);
const response = planStep
? plannedHandoffResponse(request, planStep)
: deps.evalPlan && deps.evalPlan.useActualSkills !== true
? templateEvalResponse(request, skill.name)
: await runSkillOnce({
chain,
depth,
history,
request,
skill,
});
const normalized = normalizeControlTokens(response, skill.name);
const verified = await applyRuntimeVerifier(
{
chain,
depth,
history,
request,
response: normalized,
skill,
},
deps,
);
const escalated = applyEscalationTrigger({
budget,
request,
response: verified,
skill,
});
const annotated = withHandoffMetadata(
escalated,
chain,
depth,
lastHandoff,
);
const handoff = readHandoffRequest(annotated);
if (!handoff) {
return annotated;
}
const next = prepareHandoff(
{
budget,
chain,
currentSkill: skill.name,
depth,
evalPlanIndex: input.evalPlanIndex ?? 0,
handoff,
history,
planStepUsed: Boolean(planStep),
request,
response: annotated,
},
deps,
);
if (!next) {
const fallbackHandoff: HandoffRequest = {
...handoff,
targetAgent: "supervisor",
};
return withHandoffMetadata(
makeSupervisorFallback(request, handoff.reason),
[...chain, "supervisor"],
depth + 1,
fallbackHandoff,
);
}
const maxDepth = deps.evalPlan?.maxDepth ?? 8;
if (next.depth > maxDepth) {
const fallbackHandoff: HandoffRequest = {
confidence: 0.4,
evidenceRefs: [],
expectedOutput: "text",
reason: "max-hop",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "Dispatch exceeded max loop depth.",
};
const fallbackChain =
chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"];
return withHandoffMetadata(
makeSupervisorFallback(request, "max-hop"),
fallbackChain,
fallbackChain.length - 1,
fallbackHandoff,
);
}
return dispatch(next, deps);
},
);
}
export function withHandoffMetadata(
response: AgentResponse,
chain: string[],
depth: number,
lastHandoff?: HandoffRequest,
): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffChain: chain,
handoffDepth: depth,
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
},
};
}
export function normalizeControlTokens(
response: AgentResponse,
currentSkill: string,
): AgentResponse {
const parsed = parseControlTokens(response.text);
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
const metadata = { ...(response.metadata ?? {}) };
if (handoff?.kind === "handoff") {
metadata.handoffRequest = {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: handoff.reason,
risk: "medium",
targetAgent: handoff.targetAgent,
taskSpec: `Continue handling from ${currentSkill}`,
} satisfies HandoffRequest;
}
return {
...response,
metadata,
text: parsed.stripped,
};
}
export function readHandoffRequest(
response: AgentResponse,
): HandoffRequest | null {
const value = response.metadata?.["handoffRequest"];
if (!value || typeof value !== "object") {
return null;
}
const handoff = value as Partial<HandoffRequest>;
if (
typeof handoff.targetAgent !== "string" ||
typeof handoff.reason !== "string" ||
typeof handoff.taskSpec !== "string" ||
typeof handoff.confidence !== "number" ||
typeof handoff.expectedOutput !== "string"
) {
return null;
}
return {
confidence: handoff.confidence,
evidenceRefs: Array.isArray(handoff.evidenceRefs)
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
: [],
expectedOutput: handoff.expectedOutput,
reason: handoff.reason,
risk:
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
? handoff.risk
: "medium",
targetAgent: handoff.targetAgent,
taskSpec: handoff.taskSpec,
...(handoff.userVisibleMessage
? { userVisibleMessage: handoff.userVisibleMessage }
: {}),
};
}
export function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
if (verdict === "same-agent-loop") {
return "same-agent-revisit";
}
if (verdict === "max-hops") {
return "max-hop";
}
return "policy-risk";
}
async function runSkillOnce(input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
skill: Skill;
}): Promise<AgentResponse> {
return withSkillSpan(
input.skill.name,
{
"agent.handoff_chain": input.chain.join(">"),
"agent.handoff_depth": input.depth,
"agent.skill": input.skill.name,
"agent.trigger": input.request.trigger,
},
() => input.skill.handle(input.request, input.history),
);
}
async function applyRuntimeVerifier(
input: {
chain: string[];
depth: number;
history: ConversationMessage[];
request: AgentRequest;
response: AgentResponse;
skill: Skill;
},
deps: DispatchDependencies,
): Promise<AgentResponse> {
if (!deps.shouldApplyRuntimeVerifier()) {
return input.response;
}
const judgeConfig = deps.getSkillJudgeConfig ?? defaultGetSkillJudgeConfig;
const verifier = judgeConfig(input.skill.name).runtimeVerifier;
if (!verifier) {
return input.response;
}
const verdict = await verifier({
request: input.request,
response: input.response,
skill: input.skill.name,
});
if (verdict === "pass") {
return input.response;
}
deps.onValidatorFail?.({ skill: input.skill.name, stage: verdict });
if (verdict === "retry" && input.depth === 0) {
const retryResponse = await runSkillOnce({
...input,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
runtimeVerifierRetry: true,
},
},
});
const normalizedRetry = normalizeControlTokens(
retryResponse,
input.skill.name,
);
if (readHandoffRequest(normalizedRetry)) {
return normalizedRetry;
}
const retryVerdict = await verifier({
request: input.request,
response: normalizedRetry,
skill: input.skill.name,
});
if (retryVerdict === "pass") {
return {
...normalizedRetry,
metadata: {
...(normalizedRetry.metadata ?? {}),
runtimeVerifier: "retry-pass",
},
};
}
deps.onValidatorFail?.({
skill: input.skill.name,
stage: `retry-${retryVerdict}`,
});
return withRuntimeVerifierHandoff(normalizedRetry);
}
return withRuntimeVerifierHandoff(input.response);
}
function withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: "validation-failed",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "Runtime verifier rejected the response.",
} satisfies HandoffRequest,
},
};
}
function applyEscalationTrigger(input: {
budget: BudgetTracker;
request: AgentRequest;
response: AgentResponse;
skill: Skill;
}): AgentResponse {
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
return input.response;
}
const decision = decideEscalation({
budget: input.budget.snapshot(),
capabilityCard: input.skill.capabilityCard?.(),
classified: { confidence: input.response.confidence },
targetAgent: "supervisor",
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
});
if (!decision.handoff) {
return input.response;
}
return {
...input.response,
metadata: {
...(input.response.metadata ?? {}),
handoffRequest: decision.handoff,
},
};
}
function prepareHandoff(
input: {
budget: BudgetTracker;
chain: string[];
currentSkill: string;
depth: number;
evalPlanIndex: number;
handoff: HandoffRequest;
history: ConversationMessage[];
planStepUsed: boolean;
request: AgentRequest;
response: AgentResponse;
},
deps: DispatchDependencies,
): DispatchInput | null {
const target = resolveHandoffTarget(input, deps);
const nextSkill = deps.selectSkillForRequest(deps.skills, {
...input.request,
skill: target.targetAgent,
trigger: "handoff",
});
if (!nextSkill) {
return null;
}
input.budget.recordHandoff(nextSkill.name);
deps.onHandoff?.({
from: input.currentSkill,
reason: target.reason,
to: nextSkill.name,
});
const nextChain = [...input.chain, nextSkill.name];
const lastHandoff: HandoffRequest = {
...input.handoff,
reason: target.reason,
targetAgent: nextSkill.name,
};
const packet = buildHandoffPacket({
handoff: lastHandoff,
history: input.history,
request: input.request,
response: input.response,
sourceChain: nextChain,
state: {
source: input.currentSkill,
},
});
return {
budget: input.budget,
chain: nextChain,
depth: input.depth + 1,
evalPlanIndex: input.evalPlanIndex + (input.planStepUsed ? 1 : 0),
history: input.history,
lastHandoff,
request: {
...input.request,
context: {
...(input.request.context ?? {}),
handoffPacket: packet,
},
skill: nextSkill.name,
trigger: "handoff",
},
skill: nextSkill,
};
}
function resolveHandoffTarget(
input: {
budget: BudgetTracker;
currentSkill: string;
handoff: HandoffRequest;
},
deps: DispatchDependencies,
): { reason: HandoffReason; targetAgent: string } {
const requestedTarget = input.handoff.targetAgent;
const allowed =
deps.transitionMatrix?.isAllowed(input.currentSkill, requestedTarget) ?? true;
if (!allowed) {
return {
reason: "policy-risk",
targetAgent: "supervisor",
};
}
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
if (verdict === "ok") {
return {
reason: input.handoff.reason,
targetAgent: requestedTarget,
};
}
if (verdict === "same-agent-loop") {
deps.onHandoffLoop?.({ from: input.currentSkill });
}
return {
reason: budgetVerdictToReason(verdict),
targetAgent: "supervisor",
};
}
function makeSupervisorFallback(
request: AgentRequest,
reason: HandoffReason,
): AgentResponse {
return {
confidence: 0.4,
metadata: {
supervisorFallbackReason: reason,
},
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
};
}
function getCurrentPlanStep(
input: DispatchInput,
deps: DispatchDependencies,
): EvalDispatchPlanStep | undefined {
const index = input.evalPlanIndex ?? 0;
const step = deps.evalPlan?.steps[index];
return step?.from === input.skill.name ? step : undefined;
}
function plannedHandoffResponse(
request: AgentRequest,
step: EvalDispatchPlanStep,
): AgentResponse {
return {
confidence: step.confidence ?? 0.72,
metadata: {
handoffRequest: {
confidence: step.confidence ?? 0.72,
evidenceRefs: [],
expectedOutput: step.expectedOutput ?? "text",
reason: step.reason,
risk: step.risk ?? "medium",
targetAgent: step.to,
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
} satisfies HandoffRequest,
},
requestId: request.requestId,
source: "template",
text: `handoff ${step.from} to ${step.to}`,
};
}
function templateEvalResponse(
request: AgentRequest,
skillName: string,
): AgentResponse {
return {
confidence: 1,
requestId: request.requestId,
source: "template",
text: `handoff eval final: ${skillName}`,
};
}
+12
View File
@@ -229,6 +229,18 @@ export function getSupervisorReviewResolutionTtlSec(): number {
); );
} }
export function getSupervisorResolutionsStreamMaxLen(): number {
return getPositiveNumber("SUPERVISOR_RESOLUTIONS_STREAM_MAXLEN", 100_000);
}
export function getSupervisorLlmEnabled(): boolean {
return getBoolean("AGENT_SUPERVISOR_LLM_ENABLED", true);
}
export function getSupervisorLlmConfidenceThreshold(): number {
return getRatioNumber("AGENT_SUPERVISOR_LLM_CONFIDENCE_THRESHOLD", 0.6);
}
export function getEvalCalibrationMinCorr(): number { export function getEvalCalibrationMinCorr(): number {
return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5); return getRatioNumber("EVAL_CALIBRATION_MIN_CORR", 0.5);
} }
+68 -337
View File
@@ -1,16 +1,10 @@
import { BudgetTracker, type BudgetVerdict } from "@agent/budget";
import { buildHandoffPacket } from "@agent/handoff-packet";
import { selectSkillForRequest } from "@agent/agent"; import { selectSkillForRequest } from "@agent/agent";
import type { import { BudgetTracker } from "@agent/budget";
AgentRequest, import { dispatch, type EvalDispatchPlan, type EvalDispatchPlanStep } from "@agent/dispatch";
AgentResponse, import type { AgentRequest, AgentResponse, ConversationMessage, HandoffReason } from "@agent/types";
ConversationMessage,
HandoffReason,
HandoffRequest,
} from "@agent/types";
import { parseControlTokens } from "@skills/control-token-parser";
import { decide as decideEscalation } from "@skills/escalation/trigger";
import type { Skill } from "@skills/skill"; import type { Skill } from "@skills/skill";
import type { TransitionMatrix } from "@skills/transition-matrix";
import { getSkillJudgeConfig } from "./judges/skill-config.js";
export interface EvalDispatchResult { export interface EvalDispatchResult {
chain: string[]; chain: string[];
@@ -18,29 +12,43 @@ export interface EvalDispatchResult {
response: AgentResponse; response: AgentResponse;
} }
interface EvalHandoffPlanStep {
confidence?: number | undefined;
expectedOutput?: string | undefined;
from: string;
reason: HandoffReason;
risk?: HandoffRequest["risk"] | undefined;
taskSpec?: string | undefined;
to: string;
}
function asRecord(value: unknown): Record<string, unknown> { function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null return typeof value === "object" && value !== null
? (value as Record<string, unknown>) ? (value as Record<string, unknown>)
: {}; : {};
} }
function readPlan(request: AgentRequest): EvalHandoffPlanStep[] { const HANDOFF_REASONS = [
"out-of-scope",
"missing-tool",
"low-confidence",
"validation-failed",
"same-agent-revisit",
"max-hop",
"high-risk-action",
"policy-risk",
] as const;
function isHandoffReason(value: unknown): value is HandoffReason {
return HANDOFF_REASONS.includes(value as HandoffReason);
}
function readPositiveNumber(
context: Record<string, unknown> | undefined,
key: string,
fallback: number,
): number {
const value = context?.[key];
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
}
function readPlan(request: AgentRequest): EvalDispatchPlanStep[] {
const rawPlan = asRecord(request.context)["evalHandoffPlan"]; const rawPlan = asRecord(request.context)["evalHandoffPlan"];
if (!Array.isArray(rawPlan)) { if (!Array.isArray(rawPlan)) {
return []; return [];
} }
return rawPlan.flatMap((item): EvalHandoffPlanStep[] => { return rawPlan.flatMap((item): EvalDispatchPlanStep[] => {
const record = asRecord(item); const record = asRecord(item);
const from = record["from"]; const from = record["from"];
const to = record["to"]; const to = record["to"];
@@ -55,14 +63,8 @@ function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
return [ return [
{ {
confidence: confidence: typeof record["confidence"] === "number" ? record["confidence"] : undefined,
typeof record["confidence"] === "number" expectedOutput: typeof record["expectedOutput"] === "string" ? record["expectedOutput"] : undefined,
? record["confidence"]
: undefined,
expectedOutput:
typeof record["expectedOutput"] === "string"
? record["expectedOutput"]
: undefined,
from, from,
reason, reason,
risk: risk:
@@ -71,198 +73,26 @@ function readPlan(request: AgentRequest): EvalHandoffPlanStep[] {
record["risk"] === "high" record["risk"] === "high"
? record["risk"] ? record["risk"]
: undefined, : undefined,
taskSpec: taskSpec: typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
typeof record["taskSpec"] === "string" ? record["taskSpec"] : undefined,
to, to,
}, },
]; ];
}); });
} }
function isHandoffReason(value: unknown): value is HandoffReason { function asStringArray(value: unknown): string[] {
return ( return Array.isArray(value)
value === "out-of-scope" || ? value.filter((item): item is string => typeof item === "string")
value === "missing-tool" || : [];
value === "low-confidence" ||
value === "validation-failed" ||
value === "same-agent-revisit" ||
value === "max-hop" ||
value === "high-risk-action" ||
value === "policy-risk"
);
}
function readPositiveNumber(
context: Record<string, unknown> | undefined,
key: string,
fallback: number,
): number {
const value = context?.[key];
return typeof value === "number" && Number.isFinite(value) && value > 0
? value
: fallback;
}
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
if (verdict === "same-agent-loop") {
return "same-agent-revisit";
}
if (verdict === "max-hops") {
return "max-hop";
}
return "policy-risk";
}
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
const value = response.metadata?.["handoffRequest"];
if (typeof value !== "object" || value === null) {
return null;
}
const request = value as Partial<HandoffRequest>;
if (
typeof request.targetAgent !== "string" ||
typeof request.reason !== "string" ||
typeof request.taskSpec !== "string"
) {
return null;
}
return {
confidence:
typeof request.confidence === "number" ? request.confidence : response.confidence,
evidenceRefs: Array.isArray(request.evidenceRefs)
? request.evidenceRefs.filter((item): item is string => typeof item === "string")
: [],
expectedOutput:
typeof request.expectedOutput === "string"
? request.expectedOutput
: "text",
reason: request.reason,
risk:
request.risk === "low" || request.risk === "medium" || request.risk === "high"
? request.risk
: "medium",
targetAgent: request.targetAgent,
taskSpec: request.taskSpec,
...(request.userVisibleMessage
? { userVisibleMessage: request.userVisibleMessage }
: {}),
};
}
function normalizeControlTokens(
response: AgentResponse,
currentSkill: string,
): AgentResponse {
const parsed = parseControlTokens(response.text);
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
if (handoff?.kind !== "handoff") {
return {
...response,
text: parsed.stripped,
};
}
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: {
confidence: response.confidence,
evidenceRefs: [],
expectedOutput: "text",
reason: handoff.reason,
risk: "medium",
targetAgent: handoff.targetAgent,
taskSpec: `Continue handling from ${currentSkill}`,
} satisfies HandoffRequest,
},
text: parsed.stripped,
};
}
function withHandoffMetadata(
response: AgentResponse,
chain: string[],
depth: number,
lastHandoff: HandoffRequest | undefined,
): AgentResponse {
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffChain: chain,
handoffDepth: depth,
...(lastHandoff ? { handoffLastRequest: lastHandoff } : {}),
},
};
}
function plannedHandoffResponse(
request: AgentRequest,
step: EvalHandoffPlanStep,
): AgentResponse {
return {
confidence: step.confidence ?? 0.72,
metadata: {
handoffRequest: {
confidence: step.confidence ?? 0.72,
evidenceRefs: [],
expectedOutput: step.expectedOutput ?? "text",
reason: step.reason,
risk: step.risk ?? "medium",
targetAgent: step.to,
taskSpec: step.taskSpec ?? `Continue from ${step.from} to ${step.to}`,
} satisfies HandoffRequest,
},
requestId: request.requestId,
source: "template",
text: `handoff ${step.from} to ${step.to}`,
};
}
async function runActualSkill(
request: AgentRequest,
history: ConversationMessage[],
skill: Skill,
budget: BudgetTracker,
): Promise<AgentResponse> {
const response = normalizeControlTokens(
await skill.handle(request, history),
skill.name,
);
if (!response.text.trim() || readHandoffRequest(response)) {
return response;
}
const decision = decideEscalation({
budget: budget.snapshot(),
capabilityCard: skill.capabilityCard?.(),
classified: { confidence: response.confidence },
targetAgent: "supervisor",
taskSpec: `Review ${skill.name} response for ${request.trigger}.`,
});
if (!decision.handoff) {
return response;
}
return {
...response,
metadata: {
...(response.metadata ?? {}),
handoffRequest: decision.handoff,
},
};
} }
export async function runEvalDispatch(input: { export async function runEvalDispatch(input: {
history?: ConversationMessage[] | undefined; history?: ConversationMessage[] | undefined;
request: AgentRequest; request: AgentRequest;
skills: readonly Skill[]; skills: readonly Skill[];
transitionMatrix?: TransitionMatrix | undefined;
}): Promise<EvalDispatchResult> { }): Promise<EvalDispatchResult> {
const context = input.request.context; const context = input.request.context;
const plan = readPlan(input.request);
const useActualSkills = asRecord(context)["evalUseActualSkills"] === true;
const budget = new BudgetTracker({ const budget = new BudgetTracker({
enabled: true, enabled: true,
maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5), maxCostUsd: readPositiveNumber(context, "evalBudgetMaxCostUsd", 0.5),
@@ -274,139 +104,40 @@ export async function runEvalDispatch(input: {
), ),
maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20), maxToolCalls: readPositiveNumber(context, "evalBudgetMaxToolCalls", 20),
}); });
const skill = selectSkillForRequest(input.skills, input.request);
let request = input.request;
let skill = selectSkillForRequest(input.skills, request);
if (!skill) { if (!skill) {
throw new Error(`No skill can handle eval request: ${request.skill}`); throw new Error(`No skill can handle eval request: ${input.request.skill}`);
} }
let chain = [skill.name];
let lastHandoff: HandoffRequest | undefined;
let planIndex = 0;
budget.recordVisit(skill.name); budget.recordVisit(skill.name);
for (let depth = 0; depth <= 8; depth++) { const plan: EvalDispatchPlan = {
const planned = plan[planIndex]?.from === skill.name ? plan[planIndex] : undefined; maxDepth: readPositiveNumber(context, "evalMaxDispatchDepth", 8),
const response = planned steps: readPlan(input.request),
? plannedHandoffResponse(request, planned) useActualSkills: asRecord(context)["evalUseActualSkills"] === true,
: useActualSkills
? await runActualSkill(request, input.history ?? [], skill, budget)
: {
confidence: 1,
requestId: request.requestId,
source: "template" as const,
text: `handoff eval final: ${skill.name}`,
};
if (planned) {
planIndex += 1;
}
const handoff = readHandoffRequest(response);
if (!handoff) {
const finalResponse = withHandoffMetadata(
response,
chain,
chain.length - 1,
lastHandoff,
);
return {
chain,
finalSkill: skill.name,
response: finalResponse,
};
}
const verdict = budget.assertBeforeHandoff(handoff.targetAgent);
const resolved: HandoffRequest =
verdict === "ok"
? handoff
: {
...handoff,
reason: budgetVerdictToReason(verdict),
targetAgent: "supervisor",
};
const nextSkill = selectSkillForRequest(input.skills, {
...request,
skill: resolved.targetAgent,
trigger: "handoff",
});
if (!nextSkill) {
const fallbackChain = [...chain, "supervisor"];
const fallbackHandoff = {
...resolved,
targetAgent: "supervisor",
};
return {
chain: fallbackChain,
finalSkill: "supervisor",
response: withHandoffMetadata(
{
confidence: 0.4,
requestId: request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
},
fallbackChain,
fallbackChain.length - 1,
fallbackHandoff,
),
};
}
budget.recordHandoff(nextSkill.name);
const nextChain = [...chain, nextSkill.name];
lastHandoff = {
...resolved,
targetAgent: nextSkill.name,
};
const packet = buildHandoffPacket({
handoff: lastHandoff,
history: input.history,
request,
response,
sourceChain: nextChain,
state: {
source: skill.name,
},
});
request = {
...request,
context: {
...(request.context ?? {}),
handoffPacket: packet,
},
skill: nextSkill.name,
trigger: "handoff",
};
skill = nextSkill;
chain = nextChain;
}
const finalHandoff: HandoffRequest = {
confidence: 0.4,
evidenceRefs: [],
expectedOutput: "text",
reason: "max-hop",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "Eval dispatch exceeded max loop depth.",
}; };
const fallbackChain = chain.at(-1) === "supervisor" ? chain : [...chain, "supervisor"]; const response = await dispatch(
{
budget,
chain: [skill.name],
depth: 0,
history: input.history ?? [],
request: input.request,
skill,
},
{
evalPlan: plan,
getSkillJudgeConfig,
selectSkillForRequest,
shouldApplyRuntimeVerifier: () => false,
skills: input.skills,
transitionMatrix: input.transitionMatrix,
},
);
const chain = asStringArray(response.metadata?.["handoffChain"]);
return { return {
chain: fallbackChain, chain,
finalSkill: "supervisor", finalSkill: chain.at(-1) ?? skill.name,
response: withHandoffMetadata( response,
{
confidence: 0.4,
requestId: input.request.requestId,
source: "template",
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
},
fallbackChain,
fallbackChain.length - 1,
finalHandoff,
),
}; };
} }
+108
View File
@@ -0,0 +1,108 @@
import { describe, expect, it } from "vitest";
import type { StreamEntry } from "@common/garnet";
import { BACKGROUND_STREAM_KEY } from "./shadow-bus.js";
import { loadSupervisorOutcomes } from "./router-training-signals.js";
import {
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
SUPERVISOR_REVIEW_STREAM_KEY,
} from "@skills/supervisor/review-queue";
class FakeGarnet {
constructor(private readonly streams: Record<string, StreamEntry[]>) {}
readStreamRange(key: string): Promise<StreamEntry[]> {
return Promise.resolve(this.streams[key] ?? []);
}
}
describe("loadSupervisorOutcomes", () => {
it("joins supervisor review entries, resolution audit, and shadow eval by requestId", async () => {
const garnet = new FakeGarnet({
[BACKGROUND_STREAM_KEY]: [
{
id: "10-0",
message: {
classified: "{}",
createdAt: "2026-05-18T00:00:30.000Z",
derived: "{}",
input: JSON.stringify({
request: {
chatroomId: "room-1",
nickname: "tester",
requestId: "request-1",
skill: "onboarding",
trigger: "user-message",
userId: "user-1",
},
}),
output: JSON.stringify({
confidence: 1,
requestId: "request-1",
source: "template",
text: "ok",
}),
skill: "onboarding",
},
},
],
[SUPERVISOR_RESOLUTIONS_STREAM_KEY]: [
{
id: "2-0",
message: {
action: "manual-reply",
resolvedAt: "2026-05-18T00:02:00.000Z",
resolvedBy: "operator@example.com",
reviewId: "1-0",
type: "resolution",
},
},
],
[SUPERVISOR_REVIEW_STREAM_KEY]: [
{
id: "1-0",
message: {
chatroomId: "room-1",
packet: JSON.stringify({
constraints: [],
currentState: {},
evidence: [],
expectedOutputSchema: "text",
handoffReason: "low-confidence",
sourceChain: ["onboarding", "supervisor"],
summary: "unclear",
userGoal: "help",
}),
queuedAt: "2026-05-18T00:01:00.000Z",
reason: "low-confidence",
requestId: "request-1",
target: "human-review",
userId: "user-1",
},
},
],
});
await expect(
loadSupervisorOutcomes(
garnet,
new Date("2026-05-18T00:00:00.000Z"),
new Date("2026-05-18T00:03:00.000Z"),
),
).resolves.toMatchObject([
{
reason: "low-confidence",
resolution: {
action: "manual-reply",
},
reviewId: "1-0",
shadowEvalEntry: {
request: {
requestId: "request-1",
},
},
sourceChain: ["onboarding", "supervisor"],
},
]);
});
});
+177
View File
@@ -0,0 +1,177 @@
import type { HandoffReason } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet";
import {
parseSupervisorReviewEntry,
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
SUPERVISOR_REVIEW_STREAM_KEY,
type SupervisorReviewAction,
type SupervisorReviewResolution,
} from "@skills/supervisor/review-queue";
import { BACKGROUND_STREAM_KEY, type ShadowEvalEnvelope } from "./shadow-bus.js";
export interface SupervisorOutcome {
queuedAt: string;
reason: HandoffReason;
resolution: {
action: SupervisorReviewAction;
resolvedAt: string;
};
reviewId: string;
shadowEvalEntry?: ShadowEvalEnvelope | undefined;
sourceChain: string[];
}
function parseJson<T>(value: string | undefined): T | undefined {
if (!value?.trim()) {
return undefined;
}
try {
return JSON.parse(value) as T;
} catch {
return undefined;
}
}
function isWithinIso(
value: string,
since: Date,
until: Date,
): boolean {
const time = Date.parse(value);
return (
Number.isFinite(time) &&
time >= since.getTime() &&
time <= until.getTime()
);
}
function parseShadowEntry(
entry: StreamEntry,
): ShadowEvalEnvelope | null {
const input = parseJson<{ request?: ShadowEvalEnvelope["request"] }>(
entry.message["input"],
);
const response = parseJson<ShadowEvalEnvelope["response"]>(
entry.message["output"],
);
const skill = entry.message["skill"];
const createdAt = entry.message["createdAt"];
if (!input?.request || !response || !skill || !createdAt) {
return null;
}
return {
classified: parseJson(entry.message["classified"]) ?? {},
createdAt,
derived: parseJson(entry.message["derived"]) ?? {},
request: input.request,
response,
skill,
...(entry.message["latencyMs"]
? { latencyMs: Number(entry.message["latencyMs"]) }
: {}),
...(entry.message["process"]
? { process: parseJson(entry.message["process"]) }
: {}),
};
}
function isSupervisorReviewAction(
value: string | undefined,
): value is SupervisorReviewAction {
return value === "manual-reply" || value === "reroute" || value === "dismiss";
}
function parseResolutionEntry(
entry: StreamEntry,
): { resolution: SupervisorReviewResolution; reviewId: string } | null {
const reviewId = entry.message["reviewId"];
const resolvedAt = entry.message["resolvedAt"];
const resolvedBy = entry.message["resolvedBy"];
const action = entry.message["action"];
if (
!reviewId ||
!resolvedAt ||
!resolvedBy ||
!isSupervisorReviewAction(action)
) {
return null;
}
return {
resolution: {
action,
resolvedAt,
resolvedBy,
...(entry.message["note"] ? { note: entry.message["note"] } : {}),
},
reviewId,
};
}
function latestResolutionsByReviewId(
entries: StreamEntry[],
): Map<string, SupervisorReviewResolution> {
const byReviewId = new Map<string, SupervisorReviewResolution>();
for (const entry of entries) {
const parsed = parseResolutionEntry(entry);
if (!parsed) {
continue;
}
const current = byReviewId.get(parsed.reviewId);
if (!current || parsed.resolution.resolvedAt > current.resolvedAt) {
byReviewId.set(parsed.reviewId, parsed.resolution);
}
}
return byReviewId;
}
export async function loadSupervisorOutcomes(
garnetClient: Pick<GarnetClient, "readStreamRange">,
since: Date,
until: Date,
): Promise<SupervisorOutcome[]> {
const [reviewEntries, resolutionEntries, shadowEntries] = await Promise.all([
garnetClient.readStreamRange(SUPERVISOR_REVIEW_STREAM_KEY, "-", "+"),
garnetClient.readStreamRange(SUPERVISOR_RESOLUTIONS_STREAM_KEY, "-", "+"),
garnetClient.readStreamRange(BACKGROUND_STREAM_KEY, "-", "+"),
]);
const shadowByRequestId = new Map(
shadowEntries.flatMap((entry) => {
const shadow = parseShadowEntry(entry);
return shadow ? [[shadow.request.requestId, shadow] as const] : [];
}),
);
const resolutionByReviewId = latestResolutionsByReviewId(resolutionEntries);
const outcomes: SupervisorOutcome[] = [];
for (const entry of reviewEntries) {
const review = parseSupervisorReviewEntry(entry);
if (!review || !isWithinIso(review.queuedAt, since, until)) {
continue;
}
const resolution = resolutionByReviewId.get(review.id);
if (!resolution || !isWithinIso(resolution.resolvedAt, since, until)) {
continue;
}
outcomes.push({
queuedAt: review.queuedAt,
reason: review.reason,
resolution: {
action: resolution.action,
resolvedAt: resolution.resolvedAt,
},
reviewId: review.id,
shadowEvalEntry: shadowByRequestId.get(review.requestId),
sourceChain: review.packet?.sourceChain ?? [],
});
}
return outcomes;
}
export const ROUTER_TRAINING_SIGNAL_STREAMS = {
background: BACKGROUND_STREAM_KEY,
resolutions: SUPERVISOR_RESOLUTIONS_STREAM_KEY,
reviews: SUPERVISOR_REVIEW_STREAM_KEY,
} as const;
+1 -1
View File
@@ -6,7 +6,7 @@ import {
getShadowEvalSamplingRate, getShadowEvalSamplingRate,
} from "../config.js"; } from "../config.js";
const BACKGROUND_STREAM_KEY = "lolympus-agent:background"; export const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
export interface ShadowEvalEnvelope { export interface ShadowEvalEnvelope {
classified: unknown; classified: unknown;
+1
View File
@@ -262,6 +262,7 @@ async function bootstrap(): Promise<void> {
natsClient, natsClient,
roomContextService, roomContextService,
shadowEvalBus, shadowEvalBus,
skillRegistry,
}), }),
); );
} }
+34 -24
View File
@@ -25,31 +25,41 @@ describe("operations review CLI formatters", () => {
it("formats packet details for show output", () => { it("formats packet details for show output", () => {
expect( expect(
formatReviewShow({ formatReviewShow(
chatroomId: "room-1", {
id: "1-0", chatroomId: "room-1",
packet: { id: "1-0",
constraints: [], packet: {
currentState: {}, constraints: [],
evidence: [ currentState: {},
{ evidence: [
id: "e1", {
kind: "message", id: "e1",
summary: "unclear message", kind: "message",
}, summary: "unclear message",
], },
expectedOutputSchema: "text", ],
handoffReason: "low-confidence", expectedOutputSchema: "text",
sourceChain: ["onboarding", "supervisor"], handoffReason: "low-confidence",
summary: "needs review", sourceChain: ["onboarding", "supervisor"],
userGoal: "의도 확인", summary: "needs review",
userGoal: "의도 확인",
},
queuedAt: "2026-05-18T00:00:00.000Z",
reason: "low-confidence",
requestId: "request-1",
target: "human-review",
userId: "user-1",
}, },
queuedAt: "2026-05-18T00:00:00.000Z", [
reason: "low-confidence", {
requestId: "request-1", action: "manual-reply",
target: "human-review", note: "handled",
userId: "user-1", resolvedAt: "2026-05-18T01:00:00.000Z",
}), resolvedBy: "operator@example.com",
},
],
),
).toContain("sourceChain: onboarding > supervisor"); ).toContain("sourceChain: onboarding > supervisor");
}); });
+23 -3
View File
@@ -5,12 +5,14 @@ import { loadEnvironmentFromNodeEnv } from "@common/env-loader";
import { connectGarnetForCli } from "@common/garnet-cli"; import { connectGarnetForCli } from "@common/garnet-cli";
import { initLogger } from "@obs/logger"; import { initLogger } from "@obs/logger";
import { import {
getResolutionHistory,
getSupervisorReview, getSupervisorReview,
getSupervisorReviewStats, getSupervisorReviewStats,
listSupervisorReview, listSupervisorReview,
resolveSupervisorReview, resolveSupervisorReview,
type SupervisorReviewAction, type SupervisorReviewAction,
type SupervisorReviewItem, type SupervisorReviewItem,
type SupervisorReviewResolution,
type SupervisorReviewStats, type SupervisorReviewStats,
} from "@skills/supervisor/review-queue"; } from "@skills/supervisor/review-queue";
import { import {
@@ -95,12 +97,24 @@ export function formatReviewList(items: SupervisorReviewItem[]): string {
return rows.map((row) => row.join("\t")).join("\n"); return rows.map((row) => row.join("\t")).join("\n");
} }
export function formatReviewShow(item: SupervisorReviewItem): string { export function formatReviewShow(
item: SupervisorReviewItem,
history?: SupervisorReviewResolution[],
): string {
const packet = item.packet; const packet = item.packet;
const evidence = const evidence =
packet?.evidence packet?.evidence
.map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`) .map((entry) => `${entry.id}:${entry.kind}:${entry.summary}`)
.join("\n") || "-"; .join("\n") || "-";
const historyLines =
history?.map((resolution) =>
[
`${resolution.resolvedAt} ${resolution.action} by ${resolution.resolvedBy}`,
resolution.note ? `note=${resolution.note}` : null,
]
.filter((value): value is string => Boolean(value))
.join(" "),
) ?? [];
return [ return [
`id: ${item.id}`, `id: ${item.id}`,
@@ -124,6 +138,9 @@ export function formatReviewShow(item: SupervisorReviewItem): string {
`note: ${stringifyOptional(item.resolution.note)}`, `note: ${stringifyOptional(item.resolution.note)}`,
].join("\n") ].join("\n")
: "[resolution]\n-", : "[resolution]\n-",
...(history
? ["[history]", ...(historyLines.length > 0 ? historyLines : ["-"])]
: []),
].join("\n"); ].join("\n");
} }
@@ -191,10 +208,13 @@ async function handleReviewShow(
console.error(`[review] entry not found: ${id}`); console.error(`[review] entry not found: ${id}`);
return; return;
} }
const history = hasFlag(flags, "history")
? await getResolutionHistory(connection.garnetClient, id)
: undefined;
console.info( console.info(
hasFlag(flags, "json") hasFlag(flags, "json")
? JSON.stringify(item, null, 2) ? JSON.stringify({ ...item, ...(history ? { history } : {}) }, null, 2)
: formatReviewShow(item), : formatReviewShow(item, history),
); );
} finally { } finally {
await connection.close(); await connection.close();
+2
View File
@@ -4,6 +4,7 @@ import type { ShadowEvalBus } from "@eval/shadow-bus";
import type { LlmClient } from "@llm/llm-client"; import type { LlmClient } from "@llm/llm-client";
import type { RoomContextService } from "@room/room-context-service"; import type { RoomContextService } from "@room/room-context-service";
import type { Skill } from "./skill.js"; import type { Skill } from "./skill.js";
import type { SkillRegistry } from "./skill-loader.js";
export interface SkillDeps { export interface SkillDeps {
garnetClient: GarnetClient; garnetClient: GarnetClient;
@@ -11,6 +12,7 @@ export interface SkillDeps {
natsClient: NatsClient; natsClient: NatsClient;
roomContextService?: RoomContextService | undefined; roomContextService?: RoomContextService | undefined;
shadowEvalBus: ShadowEvalBus; shadowEvalBus: ShadowEvalBus;
skillRegistry?: SkillRegistry | undefined;
} }
export type SkillFactory = (deps: SkillDeps) => Skill; export type SkillFactory = (deps: SkillDeps) => Skill;
+2 -2
View File
@@ -1,6 +1,6 @@
--- ---
name: supervisor name: supervisor
description: 멀티 skill handoff와 예산 초과 fallback을 처리하는 lightweight supervisor. description: LLM 기반 root cause 라우팅 supervisor.
version: 1 version: 1
priority: 10 priority: 10
domain: meta domain: meta
@@ -19,4 +19,4 @@ resources:
trustTier: T4 trustTier: T4
--- ---
규칙 기반 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다. LLM 기반 root cause 분석과 규칙 fallback을 함께 사용하는 supervisor다. skill 간 handoff가 실패하거나 budget guardrail에 걸리면 사용자에게 짧게 안내하고 review queue로 넘긴다.
@@ -0,0 +1,46 @@
import type { HandoffPacket } from "@agent/handoff-packet";
import type { SkillManifest, SkillRegistry } from "@skills/skill-loader";
function formatSkill(manifest: SkillManifest): string {
return [
`- ${manifest.name}: ${manifest.description}`,
`domain=${manifest.domain}`,
`canHandle=${manifest.canHandle.join(", ") || "*"}`,
`cannotHandle=${manifest.cannotHandle.join(", ") || "-"}`,
`risk=${manifest.riskTier}`,
].join(" | ");
}
export function buildSupervisorSystemPrompt(
registry: SkillRegistry,
_packet?: HandoffPacket,
): string {
const skills = registry
.list()
.filter((manifest) => manifest.name !== "supervisor");
return [
"너는 multi-agent 시스템의 supervisor다.",
"사용자 요청과 escalation 사유를 보고 어느 specialist skill로 보낼지 결정한다.",
"",
"[사용 가능한 specialist]",
...skills.map(formatSkill),
"",
"[응답 형식]",
'JSON으로만 응답: {"target":"<skill-name 또는 human-review>","confidence":0.0-1.0,"reason":"한 줄 진단"}',
].join("\n");
}
export function buildSupervisorUserPrompt(
packet?: HandoffPacket,
): string {
if (!packet) {
return "사용자 메시지가 명확하지 않음. 어디로 보낼지 결정해줘.";
}
return [
`[escalation reason]: ${packet.handoffReason}`,
`[user goal]: ${packet.userGoal}`,
`[summary]: ${packet.summary}`,
`[source chain]: ${packet.sourceChain.join(" -> ")}`,
].join("\n");
}
+6 -1
View File
@@ -5,5 +5,10 @@ import { SupervisorSkill } from "./skill.js";
export { SupervisorSkill } from "./skill.js"; export { SupervisorSkill } from "./skill.js";
export function createSupervisorSkill(deps: SkillDeps): Skill { export function createSupervisorSkill(deps: SkillDeps): Skill {
return new SupervisorSkill(deps.garnetClient, deps.natsClient); return new SupervisorSkill(
deps.garnetClient,
deps.natsClient,
deps.llmClient,
deps.skillRegistry,
);
} }
+48 -2
View File
@@ -4,15 +4,18 @@ import type { StreamEntry } from "@common/garnet";
import { import {
getSupervisorReview, getSupervisorReview,
getSupervisorReviewStats, getSupervisorReviewStats,
getResolutionHistory,
listSupervisorReview, listSupervisorReview,
parseSupervisorReviewEntry, parseSupervisorReviewEntry,
resolveSupervisorReview, resolveSupervisorReview,
SUPERVISOR_RESOLVED_HASH_PREFIX, SUPERVISOR_RESOLVED_HASH_PREFIX,
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
SUPERVISOR_REVIEW_STREAM_KEY, SUPERVISOR_REVIEW_STREAM_KEY,
} from "./review-queue.js"; } from "./review-queue.js";
class FakeGarnet { class FakeGarnet {
readonly hashes = new Map<string, Record<string, string>>(); readonly hashes = new Map<string, Record<string, string>>();
readonly resolutionStream: StreamEntry[] = [];
readonly stream: StreamEntry[]; readonly stream: StreamEntry[];
constructor(stream: StreamEntry[]) { constructor(stream: StreamEntry[]) {
@@ -25,8 +28,12 @@ class FakeGarnet {
end: string, end: string,
options?: { count?: number | undefined }, options?: { count?: number | undefined },
): Promise<StreamEntry[]> { ): Promise<StreamEntry[]> {
expect(key).toBe(SUPERVISOR_REVIEW_STREAM_KEY); expect([SUPERVISOR_REVIEW_STREAM_KEY, SUPERVISOR_RESOLUTIONS_STREAM_KEY]).toContain(key);
const entries = this.stream.filter((entry) => { const source =
key === SUPERVISOR_RESOLUTIONS_STREAM_KEY
? this.resolutionStream
: this.stream;
const entries = source.filter((entry) => {
const afterStart = start === "-" || entry.id >= start; const afterStart = start === "-" || entry.id >= start;
const beforeEnd = end === "+" || entry.id <= end; const beforeEnd = end === "+" || entry.id <= end;
return afterStart && beforeEnd; return afterStart && beforeEnd;
@@ -43,6 +50,16 @@ class FakeGarnet {
return Promise.resolve(); return Promise.resolve();
} }
appendStream(key: string, fields: Record<string, string>): Promise<string> {
expect(key).toBe(SUPERVISOR_RESOLUTIONS_STREAM_KEY);
const id = `${this.resolutionStream.length + 1}-0`;
this.resolutionStream.push({
id,
message: { ...fields },
});
return Promise.resolve(id);
}
expireKey(): Promise<boolean> { expireKey(): Promise<boolean> {
return Promise.resolve(true); return Promise.resolve(true);
} }
@@ -140,6 +157,35 @@ describe("supervisor review queue", () => {
}); });
}); });
it("dual-writes resolution updates to an audit stream", async () => {
const garnet = new FakeGarnet(entries);
await resolveSupervisorReview(garnet, "1-0", {
action: "manual-reply",
note: "handled",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
});
await resolveSupervisorReview(garnet, "1-0", {
action: "dismiss",
resolvedAt: "2026-05-18T03:00:00.000Z",
resolvedBy: "operator-2@example.com",
});
await expect(getResolutionHistory(garnet, "1-0")).resolves.toEqual([
{
action: "dismiss",
resolvedAt: "2026-05-18T03:00:00.000Z",
resolvedBy: "operator-2@example.com",
},
{
action: "manual-reply",
note: "handled",
resolvedAt: "2026-05-18T02:00:00.000Z",
resolvedBy: "operator@example.com",
},
]);
});
it("computes reason totals and top source chains", async () => { it("computes reason totals and top source chains", async () => {
const garnet = new FakeGarnet(entries); const garnet = new FakeGarnet(entries);
garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, { garnet.hashes.set(`${SUPERVISOR_RESOLVED_HASH_PREFIX}1-0`, {
+34 -1
View File
@@ -2,6 +2,7 @@ import type { HandoffPacket } from "@agent/handoff-packet";
import type { HandoffReason } from "@agent/types"; import type { HandoffReason } from "@agent/types";
import type { GarnetClient, StreamEntry } from "@common/garnet"; import type { GarnetClient, StreamEntry } from "@common/garnet";
import { import {
getSupervisorResolutionsStreamMaxLen,
getSupervisorReviewQueueMaxLen, getSupervisorReviewQueueMaxLen,
getSupervisorReviewResolutionTtlSec, getSupervisorReviewResolutionTtlSec,
} from "../../config.js"; } from "../../config.js";
@@ -9,6 +10,8 @@ import {
export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue"; export const SUPERVISOR_REVIEW_STREAM_KEY = "lolympus-agent:review-queue";
export const SUPERVISOR_RESOLVED_HASH_PREFIX = export const SUPERVISOR_RESOLVED_HASH_PREFIX =
"lolympus-agent:review-resolved:"; "lolympus-agent:review-resolved:";
export const SUPERVISOR_RESOLUTIONS_STREAM_KEY =
"lolympus-agent:review-resolutions";
export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss"; export type SupervisorReviewAction = "manual-reply" | "reroute" | "dismiss";
@@ -67,7 +70,7 @@ type SupervisorReviewListGarnetClient = Pick<
type SupervisorReviewResolveGarnetClient = Pick< type SupervisorReviewResolveGarnetClient = Pick<
GarnetClient, GarnetClient,
"expireKey" | "setHash" "appendStream" | "expireKey" | "setHash"
>; >;
const BASE_FIELDS = new Set([ const BASE_FIELDS = new Set([
@@ -285,6 +288,36 @@ export async function resolveSupervisorReview(
const key = resolutionKey(id); const key = resolutionKey(id);
await garnetClient.setHash(key, fields); await garnetClient.setHash(key, fields);
await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec()); await garnetClient.expireKey(key, getSupervisorReviewResolutionTtlSec());
await garnetClient.appendStream(
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
{
...fields,
reviewId: id,
type: "resolution",
},
{ maxLen: getSupervisorResolutionsStreamMaxLen() },
);
}
export async function getResolutionHistory(
garnetClient: Pick<GarnetClient, "readStreamRange">,
id: string,
): Promise<SupervisorReviewResolution[]> {
const entries = await garnetClient.readStreamRange(
SUPERVISOR_RESOLUTIONS_STREAM_KEY,
"-",
"+",
);
return entries
.flatMap((entry) => {
if (entry.message["reviewId"] !== id) {
return [];
}
const resolution = parseResolution(entry.message);
return resolution ? [resolution] : [];
})
.sort((left, right) => right.resolvedAt.localeCompare(left.resolvedAt));
} }
export async function getSupervisorReviewStats( export async function getSupervisorReviewStats(
+9
View File
@@ -0,0 +1,9 @@
import { z } from "zod";
export const supervisorRoutingSchema = z.object({
confidence: z.number().min(0).max(1),
reason: z.string().max(200),
target: z.string().min(1),
});
export type SupervisorRoutingDecision = z.infer<typeof supervisorRoutingSchema>;
+81
View File
@@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest";
import type { AlertPayload } from "@common/nats-subjects"; import type { AlertPayload } from "@common/nats-subjects";
import { buildHandoffPacket } from "@agent/handoff-packet"; import { buildHandoffPacket } from "@agent/handoff-packet";
import type { AgentRequest, AgentResponse } from "@agent/types"; import type { AgentRequest, AgentResponse } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import type { SkillRegistry } from "@skills/skill-loader";
import { SupervisorSkill } from "./skill.js"; import { SupervisorSkill } from "./skill.js";
class FakeGarnet { class FakeGarnet {
@@ -27,6 +29,44 @@ class FakeNats {
} }
} }
class FakeLlm {
chatCompletion(): Promise<{
content: string;
endpoint: string;
message: { content: string; role: "assistant" };
provider: "llama";
raw: object;
}> {
return Promise.resolve({
content: JSON.stringify({
confidence: 0.9,
reason: "광고 규칙 문의라 moderation이 적합",
target: "moderation",
}),
endpoint: "memory",
message: { content: "", role: "assistant" },
provider: "llama",
raw: {},
});
}
}
const registry = {
get: (name: string) => (name === "moderation" ? { name } : null),
list: () => [
{
canHandle: ["rules"],
cannotHandle: [],
description: "커뮤니티 규칙 처리",
domain: "community",
handoffTargets: [],
name: "moderation",
outputContract: {},
riskTier: "medium",
},
],
} as unknown as SkillRegistry;
const baseRequest: AgentRequest = { const baseRequest: AgentRequest = {
chatroomId: "room-1", chatroomId: "room-1",
nickname: "tester", nickname: "tester",
@@ -70,6 +110,7 @@ describe("SupervisorSkill", () => {
expect(response.text).toContain("운영자"); expect(response.text).toContain("운영자");
expect(response.metadata).toMatchObject({ expect(response.metadata).toMatchObject({
routedBy: "rules",
supervisorReason: "max-hop", supervisorReason: "max-hop",
suggestedTarget: "onboarding", suggestedTarget: "onboarding",
}); });
@@ -96,4 +137,44 @@ describe("SupervisorSkill", () => {
metric: "agent.supervisor_handoff", metric: "agent.supervisor_handoff",
}); });
}); });
it("uses LLM routing when the decision is confident and valid", async () => {
const garnet = new FakeGarnet();
const nats = new FakeNats();
const skill = new SupervisorSkill(
garnet,
nats,
new FakeLlm() as unknown as Pick<LlmClient, "chatCompletion">,
registry,
);
const packet = buildHandoffPacket({
handoff: {
confidence: 0.7,
evidenceRefs: [],
expectedOutput: "text",
reason: "out-of-scope",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "광고 규칙 문의",
},
request: baseRequest,
response: sourceResponse,
sourceChain: ["onboarding", "supervisor"],
});
const response = await skill.handle(
{ ...baseRequest, context: { handoffPacket: packet } },
[],
);
expect(response.metadata).toMatchObject({
routedBy: "llm",
supervisorReason: "광고 규칙 문의라 moderation이 적합",
suggestedTarget: "moderation",
});
expect(garnet.streams[0]?.fields).toMatchObject({
reason: "out-of-scope",
target: "moderation",
});
});
}); });
+75 -3
View File
@@ -4,14 +4,29 @@ import type {
ConversationMessage, ConversationMessage,
} from "@agent/types"; } from "@agent/types";
import { readHandoffPacket } from "@agent/handoff-packet"; import { readHandoffPacket } from "@agent/handoff-packet";
import type { HandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet"; import type { GarnetClient } from "@common/garnet";
import type { NatsClient } from "@common/nats"; import type { NatsClient } from "@common/nats";
import type { LlmClient } from "@llm/llm-client";
import { AlertService } from "@obs/alert"; import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics"; import { metrics } from "@obs/metrics";
import type { Skill } from "@skills/skill"; import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard } from "@skills/skill-loader"; import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
import { getLogger } from "@obs/logger";
import {
getSupervisorLlmConfidenceThreshold,
getSupervisorLlmEnabled,
} from "../../config.js";
import {
buildSupervisorSystemPrompt,
buildSupervisorUserPrompt,
} from "./capability-prompt.js";
import { appendSupervisorReview } from "./review-queue.js"; import { appendSupervisorReview } from "./review-queue.js";
import { routeSupervisorTarget } from "./routing-rules.js"; import { routeSupervisorTarget } from "./routing-rules.js";
import {
supervisorRoutingSchema,
type SupervisorRoutingDecision,
} from "./schema.js";
const CAPABILITY_CARD: SkillCapabilityCard = { const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: ["*"], canHandle: ["*"],
@@ -34,6 +49,8 @@ export class SupervisorSkill implements Skill {
constructor( constructor(
private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">, private readonly garnetClient: Pick<GarnetClient, "appendStream" | "setStringIfAbsent">,
natsClient: Pick<NatsClient, "publish">, natsClient: Pick<NatsClient, "publish">,
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
private readonly skillRegistry?: SkillRegistry | undefined,
) { ) {
this.alertService = new AlertService(garnetClient, natsClient); this.alertService = new AlertService(garnetClient, natsClient);
} }
@@ -51,7 +68,11 @@ export class SupervisorSkill implements Skill {
_conversationHistory: ConversationMessage[], _conversationHistory: ConversationMessage[],
): Promise<AgentResponse> { ): Promise<AgentResponse> {
const packet = readHandoffPacket(request); const packet = readHandoffPacket(request);
const target = routeSupervisorTarget(packet); const llmDecision = await this.routeByLlm(packet);
const useLlmRoute =
llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
const target = useLlmRoute ? llmDecision.target : routeSupervisorTarget(packet);
const reason = packet?.handoffReason ?? "out-of-scope"; const reason = packet?.handoffReason ?? "out-of-scope";
await appendSupervisorReview(this.garnetClient, { await appendSupervisorReview(this.garnetClient, {
@@ -83,7 +104,8 @@ export class SupervisorSkill implements Skill {
return { return {
confidence: 0.6, confidence: 0.6,
metadata: { metadata: {
supervisorReason: reason, routedBy: useLlmRoute ? "llm" : "rules",
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
suggestedTarget: target, suggestedTarget: target,
}, },
requestId: request.requestId, requestId: request.requestId,
@@ -91,4 +113,54 @@ export class SupervisorSkill implements Skill {
text: "확인이 필요한 내용이라 운영자에게 전달했어요.", text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
}; };
} }
private async routeByLlm(
packet?: HandoffPacket,
): Promise<SupervisorRoutingDecision | null> {
if (!getSupervisorLlmEnabled() || !this.llmClient || !this.skillRegistry) {
return null;
}
try {
const result = await this.llmClient.chatCompletion(
[
{
content: buildSupervisorSystemPrompt(this.skillRegistry, packet),
role: "system",
},
{
content: buildSupervisorUserPrompt(packet),
role: "user",
},
],
{
jsonMode: true,
maxTokens: 150,
temperature: 0,
tier: 0,
},
);
if (!result?.content.trim()) {
return null;
}
const parsed = supervisorRoutingSchema.safeParse(
JSON.parse(result.content) as unknown,
);
if (!parsed.success || !this.isKnownTarget(parsed.data.target)) {
return null;
}
return parsed.data;
} catch (error) {
getLogger().warn(
{ err: error },
"[Supervisor] LLM routing failed, falling back to rules",
);
return null;
}
}
private isKnownTarget(target: string): boolean {
return target === "human-review" || Boolean(this.skillRegistry?.get(target));
}
} }