652 lines
18 KiB
TypeScript
652 lines
18 KiB
TypeScript
import type { Msg } from "nats";
|
|
import type { ConversationStore } from "@conv/conversation-store";
|
|
import type { NatsClient } from "@common/nats";
|
|
import { getRequestLogger, runWithContext } from "@obs/context";
|
|
import { getLogger } from "@obs/logger";
|
|
import { withSkillSpan, withSpan } from "@obs/spans";
|
|
import type { Skill } from "@skills/skill";
|
|
import type { TransitionMatrix } from "@skills/transition-matrix";
|
|
import { parseControlTokens } from "@skills/control-token-parser";
|
|
import { decide as decideEscalation } from "@skills/escalation/trigger";
|
|
import { getSkillJudgeConfig } from "@eval/judges/skill-config";
|
|
import { metrics } from "@obs/metrics";
|
|
import { BudgetTracker, type BudgetVerdict } from "./budget.js";
|
|
import { buildHandoffPacket } from "./handoff-packet.js";
|
|
import type {
|
|
AgentRequest,
|
|
AgentResponse,
|
|
ConversationMessage,
|
|
HandoffReason,
|
|
HandoffRequest,
|
|
} from "./types.js";
|
|
import {
|
|
getAgentRuntimeVerifierEnabled,
|
|
getAgentRequestTimeoutMs,
|
|
getRequestSubject,
|
|
getSafetyMarginMs,
|
|
} from "../config.js";
|
|
|
|
function isValidRequest(data: unknown): data is AgentRequest {
|
|
if (typeof data !== "object" || data === null) return false;
|
|
const obj = data as Record<string, unknown>;
|
|
return (
|
|
typeof obj["requestId"] === "string" &&
|
|
typeof obj["skill"] === "string" &&
|
|
typeof obj["chatroomId"] === "string" &&
|
|
typeof obj["userId"] === "string" &&
|
|
typeof obj["nickname"] === "string" &&
|
|
typeof obj["trigger"] === "string"
|
|
);
|
|
}
|
|
|
|
export function selectSkillForRequest(
|
|
skills: readonly Skill[],
|
|
request: AgentRequest,
|
|
): Skill | null {
|
|
return (
|
|
skills
|
|
.filter((candidate) => candidate.canHandle(request))
|
|
.sort((left, right) => right.priority - left.priority)[0] ?? null
|
|
);
|
|
}
|
|
|
|
export class Agent {
|
|
private readonly skills: Skill[] = [];
|
|
private readonly logger = getLogger();
|
|
|
|
constructor(
|
|
private readonly natsClient: NatsClient,
|
|
private readonly conversationStore: ConversationStore,
|
|
private readonly transitionMatrix?: TransitionMatrix | undefined,
|
|
) {}
|
|
|
|
registerSkill(skill: Skill): void {
|
|
this.skills.push(skill);
|
|
this.logger.info({ skill: skill.name }, "✓ Skill registered");
|
|
}
|
|
|
|
async start(): Promise<void> {
|
|
const subject = getRequestSubject();
|
|
|
|
await this.natsClient.subscribeRaw(subject, async (msg: Msg) => {
|
|
await this.handleRequest(msg);
|
|
});
|
|
|
|
this.logger.info({ subject }, "✓ Agent listening on subject");
|
|
}
|
|
|
|
private async handleRequest(msg: Msg): Promise<void> {
|
|
let request: AgentRequest;
|
|
try {
|
|
const raw = JSON.parse(
|
|
new TextDecoder().decode(msg.data),
|
|
) as unknown;
|
|
if (!isValidRequest(raw)) {
|
|
this.logger.warn("[Agent] Invalid request received, skipping");
|
|
return;
|
|
}
|
|
request = raw;
|
|
} catch {
|
|
this.logger.warn("[Agent] Failed to parse request, skipping");
|
|
return;
|
|
}
|
|
|
|
await runWithContext(
|
|
{
|
|
chatroomId: request.chatroomId,
|
|
deadlineAt: new Date(
|
|
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
|
).toISOString(),
|
|
requestId: request.requestId,
|
|
skill: request.skill,
|
|
startedAt: new Date().toISOString(),
|
|
trigger: request.trigger,
|
|
userId: request.userId,
|
|
},
|
|
async () => {
|
|
const logger = getRequestLogger();
|
|
await withSpan(
|
|
"agent.handle",
|
|
{
|
|
"agent.chatroom_id": request.chatroomId,
|
|
"agent.request_id": request.requestId,
|
|
"agent.skill": request.skill,
|
|
"agent.trigger": request.trigger,
|
|
"agent.user_id": request.userId,
|
|
},
|
|
async (rootSpan) => {
|
|
logger.info(
|
|
{
|
|
nickname: request.nickname,
|
|
},
|
|
"[Agent] Request received",
|
|
);
|
|
|
|
let skill = selectSkillForRequest(this.skills, request);
|
|
if (!skill) {
|
|
const supervisorRequest: AgentRequest = {
|
|
...request,
|
|
skill: "supervisor",
|
|
trigger: "handoff",
|
|
};
|
|
skill = selectSkillForRequest(this.skills, supervisorRequest);
|
|
if (skill) {
|
|
request = supervisorRequest;
|
|
} else {
|
|
logger.info("[Agent] No skill found for request");
|
|
const response: AgentResponse = {
|
|
requestId: request.requestId,
|
|
text: "",
|
|
confidence: 0,
|
|
source: "template",
|
|
};
|
|
rootSpan.setAttribute("agent.source", "template");
|
|
await this.natsClient.respond(msg, response);
|
|
return;
|
|
}
|
|
}
|
|
|
|
try {
|
|
const history = await this.conversationStore.getHistory(
|
|
request.chatroomId,
|
|
request.userId,
|
|
);
|
|
|
|
if (request.userMessage?.trim()) {
|
|
await this.conversationStore.appendMessage(
|
|
request.chatroomId,
|
|
request.userId,
|
|
{
|
|
role: "user",
|
|
content: request.userMessage,
|
|
timestamp: new Date().toISOString(),
|
|
},
|
|
);
|
|
}
|
|
|
|
const startedAt = Date.now();
|
|
const budget = new BudgetTracker();
|
|
budget.recordVisit(skill.name);
|
|
const response = await this.dispatch({
|
|
budget,
|
|
chain: [skill.name],
|
|
depth: 0,
|
|
history,
|
|
request,
|
|
skill,
|
|
});
|
|
|
|
if (response.text.trim()) {
|
|
await this.conversationStore.appendMessage(
|
|
request.chatroomId,
|
|
request.userId,
|
|
{
|
|
role: "assistant",
|
|
content: response.text,
|
|
timestamp: new Date().toISOString(),
|
|
},
|
|
);
|
|
}
|
|
|
|
await this.natsClient.respond(msg, response);
|
|
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
|
|
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
|
|
rootSpan.setAttribute(
|
|
"agent.source",
|
|
response.text.trim() ? response.source : "empty",
|
|
);
|
|
|
|
logger.info(
|
|
{
|
|
confidence: response.confidence,
|
|
length: response.text.length,
|
|
source: response.source,
|
|
},
|
|
"[Agent] Response sent",
|
|
);
|
|
} catch (error) {
|
|
logger.error(
|
|
{
|
|
err: error,
|
|
},
|
|
`[Agent] Skill ${skill.name} failed`,
|
|
);
|
|
|
|
const errorResponse: AgentResponse = {
|
|
requestId: request.requestId,
|
|
text: "",
|
|
confidence: 0,
|
|
source: "template",
|
|
metadata: {
|
|
error:
|
|
error instanceof Error ? error.message : String(error),
|
|
},
|
|
};
|
|
rootSpan.setAttribute("agent.source", "template");
|
|
await this.natsClient.respond(msg, errorResponse);
|
|
}
|
|
},
|
|
);
|
|
},
|
|
);
|
|
}
|
|
|
|
private async dispatch(input: {
|
|
budget: BudgetTracker;
|
|
chain: string[];
|
|
depth: number;
|
|
history: ConversationMessage[];
|
|
request: AgentRequest;
|
|
skill: Skill;
|
|
}): Promise<AgentResponse> {
|
|
const { budget, chain, depth, history, request, skill } = input;
|
|
const contextBase = {
|
|
chatroomId: request.chatroomId,
|
|
deadlineAt: new Date(
|
|
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
|
).toISOString(),
|
|
requestId: request.requestId,
|
|
skill: skill.name,
|
|
startedAt: new Date().toISOString(),
|
|
trigger: request.trigger,
|
|
userId: request.userId,
|
|
};
|
|
|
|
return runWithContext(
|
|
{
|
|
...contextBase,
|
|
budget: budget.snapshot(),
|
|
handoffChain: chain,
|
|
handoffDepth: depth,
|
|
},
|
|
async () => {
|
|
const response = await this.runSkillOnce({
|
|
chain,
|
|
depth,
|
|
history,
|
|
request,
|
|
skill,
|
|
});
|
|
const normalized = this.normalizeControlTokens(response, skill.name);
|
|
const verified = await this.applyRuntimeVerifier({
|
|
chain,
|
|
depth,
|
|
history,
|
|
request,
|
|
response: normalized,
|
|
skill,
|
|
});
|
|
const escalated = this.applyEscalationTrigger({
|
|
budget,
|
|
request,
|
|
response: verified,
|
|
skill,
|
|
});
|
|
const annotated = this.withHandoffMetadata(escalated, chain, depth);
|
|
|
|
const handoff = readHandoffRequest(annotated);
|
|
if (!handoff) {
|
|
return annotated;
|
|
}
|
|
|
|
const next = this.prepareHandoff({
|
|
budget,
|
|
chain,
|
|
currentSkill: skill.name,
|
|
depth,
|
|
handoff,
|
|
history,
|
|
request,
|
|
response: annotated,
|
|
});
|
|
|
|
if (!next) {
|
|
return this.withHandoffMetadata(
|
|
this.makeSupervisorFallback(request, handoff.reason),
|
|
[...chain, "supervisor"],
|
|
depth + 1,
|
|
);
|
|
}
|
|
|
|
return this.dispatch(next);
|
|
},
|
|
);
|
|
}
|
|
|
|
private withHandoffMetadata(
|
|
response: AgentResponse,
|
|
chain: string[],
|
|
depth: number,
|
|
): AgentResponse {
|
|
return {
|
|
...response,
|
|
metadata: {
|
|
...(response.metadata ?? {}),
|
|
handoffChain: chain,
|
|
handoffDepth: depth,
|
|
},
|
|
};
|
|
}
|
|
|
|
private async runSkillOnce(input: {
|
|
chain: string[];
|
|
depth: number;
|
|
history: ConversationMessage[];
|
|
request: AgentRequest;
|
|
skill: Skill;
|
|
}): Promise<AgentResponse> {
|
|
return withSkillSpan(
|
|
input.skill.name,
|
|
{
|
|
"agent.handoff_chain": input.chain.join(">"),
|
|
"agent.handoff_depth": input.depth,
|
|
"agent.skill": input.skill.name,
|
|
"agent.trigger": input.request.trigger,
|
|
},
|
|
() => input.skill.handle(input.request, input.history),
|
|
);
|
|
}
|
|
|
|
private normalizeControlTokens(
|
|
response: AgentResponse,
|
|
currentSkill: string,
|
|
): AgentResponse {
|
|
const parsed = parseControlTokens(response.text);
|
|
const handoff = parsed.tokens.find((token) => token.kind === "handoff");
|
|
const metadata = { ...(response.metadata ?? {}) };
|
|
if (handoff?.kind === "handoff") {
|
|
metadata.handoffRequest = {
|
|
confidence: response.confidence,
|
|
evidenceRefs: [],
|
|
expectedOutput: "text",
|
|
reason: handoff.reason,
|
|
risk: "medium",
|
|
targetAgent: handoff.targetAgent,
|
|
taskSpec: `Continue handling from ${currentSkill}`,
|
|
} satisfies HandoffRequest;
|
|
}
|
|
return {
|
|
...response,
|
|
metadata,
|
|
text: parsed.stripped,
|
|
};
|
|
}
|
|
|
|
private async applyRuntimeVerifier(input: {
|
|
chain: string[];
|
|
depth: number;
|
|
history: ConversationMessage[];
|
|
request: AgentRequest;
|
|
response: AgentResponse;
|
|
skill: Skill;
|
|
}): Promise<AgentResponse> {
|
|
if (!getAgentRuntimeVerifierEnabled()) {
|
|
return input.response;
|
|
}
|
|
const verifier = getSkillJudgeConfig(input.skill.name).runtimeVerifier;
|
|
if (!verifier) {
|
|
return input.response;
|
|
}
|
|
const verdict = await verifier({
|
|
request: input.request,
|
|
response: input.response,
|
|
skill: input.skill.name,
|
|
});
|
|
if (verdict === "pass") {
|
|
return input.response;
|
|
}
|
|
metrics.validatorFailTotal.inc({ skill: input.skill.name, stage: verdict });
|
|
if (verdict === "retry" && input.depth === 0) {
|
|
const retryResponse = await this.runSkillOnce({
|
|
...input,
|
|
request: {
|
|
...input.request,
|
|
context: {
|
|
...(input.request.context ?? {}),
|
|
runtimeVerifierRetry: true,
|
|
},
|
|
},
|
|
});
|
|
const normalizedRetry = this.normalizeControlTokens(
|
|
retryResponse,
|
|
input.skill.name,
|
|
);
|
|
if (readHandoffRequest(normalizedRetry)) {
|
|
return normalizedRetry;
|
|
}
|
|
const retryVerdict = await verifier({
|
|
request: input.request,
|
|
response: normalizedRetry,
|
|
skill: input.skill.name,
|
|
});
|
|
if (retryVerdict === "pass") {
|
|
return {
|
|
...normalizedRetry,
|
|
metadata: {
|
|
...(normalizedRetry.metadata ?? {}),
|
|
runtimeVerifier: "retry-pass",
|
|
},
|
|
};
|
|
}
|
|
metrics.validatorFailTotal.inc({
|
|
skill: input.skill.name,
|
|
stage: `retry-${retryVerdict}`,
|
|
});
|
|
return this.withRuntimeVerifierHandoff(normalizedRetry);
|
|
}
|
|
return this.withRuntimeVerifierHandoff(input.response);
|
|
}
|
|
|
|
private withRuntimeVerifierHandoff(response: AgentResponse): AgentResponse {
|
|
return {
|
|
...response,
|
|
metadata: {
|
|
...(response.metadata ?? {}),
|
|
handoffRequest: {
|
|
confidence: response.confidence,
|
|
evidenceRefs: [],
|
|
expectedOutput: "text",
|
|
reason: "validation-failed",
|
|
risk: "medium",
|
|
targetAgent: "supervisor",
|
|
taskSpec: "Runtime verifier rejected the response.",
|
|
} satisfies HandoffRequest,
|
|
},
|
|
};
|
|
}
|
|
|
|
private applyEscalationTrigger(input: {
|
|
budget: BudgetTracker;
|
|
request: AgentRequest;
|
|
response: AgentResponse;
|
|
skill: Skill;
|
|
}): AgentResponse {
|
|
if (!input.response.text.trim() || readHandoffRequest(input.response)) {
|
|
return input.response;
|
|
}
|
|
|
|
const decision = decideEscalation({
|
|
budget: input.budget.snapshot(),
|
|
capabilityCard: input.skill.capabilityCard?.(),
|
|
classified: { confidence: input.response.confidence },
|
|
targetAgent: "supervisor",
|
|
taskSpec: `Review ${input.skill.name} response for ${input.request.trigger}.`,
|
|
});
|
|
if (!decision.handoff) {
|
|
return input.response;
|
|
}
|
|
|
|
return {
|
|
...input.response,
|
|
metadata: {
|
|
...(input.response.metadata ?? {}),
|
|
handoffRequest: decision.handoff,
|
|
},
|
|
};
|
|
}
|
|
|
|
private prepareHandoff(input: {
|
|
budget: BudgetTracker;
|
|
chain: string[];
|
|
currentSkill: string;
|
|
depth: number;
|
|
handoff: HandoffRequest;
|
|
history: ConversationMessage[];
|
|
request: AgentRequest;
|
|
response: AgentResponse;
|
|
}):
|
|
| {
|
|
budget: BudgetTracker;
|
|
chain: string[];
|
|
depth: number;
|
|
history: ConversationMessage[];
|
|
request: AgentRequest;
|
|
skill: Skill;
|
|
}
|
|
| null {
|
|
const target = this.resolveHandoffTarget(input);
|
|
const nextSkill = selectSkillForRequest(this.skills, {
|
|
...input.request,
|
|
skill: target.targetAgent,
|
|
trigger: "handoff",
|
|
});
|
|
if (!nextSkill) {
|
|
return null;
|
|
}
|
|
|
|
input.budget.recordHandoff(nextSkill.name);
|
|
metrics.handoffTotal.inc({
|
|
from: input.currentSkill,
|
|
reason: target.reason,
|
|
to: nextSkill.name,
|
|
});
|
|
|
|
const nextChain = [...input.chain, nextSkill.name];
|
|
const packet = buildHandoffPacket({
|
|
handoff: {
|
|
...input.handoff,
|
|
reason: target.reason,
|
|
targetAgent: nextSkill.name,
|
|
},
|
|
history: input.history,
|
|
request: input.request,
|
|
response: input.response,
|
|
sourceChain: nextChain,
|
|
state: {
|
|
source: input.currentSkill,
|
|
},
|
|
});
|
|
|
|
return {
|
|
budget: input.budget,
|
|
chain: nextChain,
|
|
depth: input.depth + 1,
|
|
history: input.history,
|
|
request: {
|
|
...input.request,
|
|
context: {
|
|
...(input.request.context ?? {}),
|
|
handoffPacket: packet,
|
|
},
|
|
skill: nextSkill.name,
|
|
trigger: "handoff",
|
|
},
|
|
skill: nextSkill,
|
|
};
|
|
}
|
|
|
|
private resolveHandoffTarget(input: {
|
|
budget: BudgetTracker;
|
|
currentSkill: string;
|
|
handoff: HandoffRequest;
|
|
}): { reason: HandoffReason; targetAgent: string } {
|
|
const requestedTarget = input.handoff.targetAgent;
|
|
const allowed = this.transitionMatrix?.isAllowed(
|
|
input.currentSkill,
|
|
requestedTarget,
|
|
) ?? true;
|
|
if (!allowed) {
|
|
return {
|
|
reason: "policy-risk",
|
|
targetAgent: "supervisor",
|
|
};
|
|
}
|
|
|
|
const verdict = input.budget.assertBeforeHandoff(requestedTarget);
|
|
if (verdict === "ok") {
|
|
return {
|
|
reason: input.handoff.reason,
|
|
targetAgent: requestedTarget,
|
|
};
|
|
}
|
|
|
|
if (verdict === "same-agent-loop") {
|
|
metrics.handoffLoopTotal.inc({ from: input.currentSkill });
|
|
}
|
|
|
|
return {
|
|
reason: budgetVerdictToReason(verdict),
|
|
targetAgent: "supervisor",
|
|
};
|
|
}
|
|
|
|
private makeSupervisorFallback(
|
|
request: AgentRequest,
|
|
reason: HandoffReason,
|
|
): AgentResponse {
|
|
return {
|
|
confidence: 0.4,
|
|
metadata: {
|
|
supervisorFallbackReason: reason,
|
|
},
|
|
requestId: request.requestId,
|
|
source: "template",
|
|
text: "확인이 필요한 내용이라 운영자에게 전달했어요.",
|
|
};
|
|
}
|
|
}
|
|
|
|
function budgetVerdictToReason(verdict: BudgetVerdict): HandoffReason {
|
|
if (verdict === "same-agent-loop") {
|
|
return "same-agent-revisit";
|
|
}
|
|
if (verdict === "max-hops") {
|
|
return "max-hop";
|
|
}
|
|
return "policy-risk";
|
|
}
|
|
|
|
function readHandoffRequest(response: AgentResponse): HandoffRequest | null {
|
|
const value = response.metadata?.["handoffRequest"];
|
|
if (!value || typeof value !== "object") {
|
|
return null;
|
|
}
|
|
const handoff = value as Partial<HandoffRequest>;
|
|
if (
|
|
typeof handoff.targetAgent !== "string" ||
|
|
typeof handoff.reason !== "string" ||
|
|
typeof handoff.taskSpec !== "string" ||
|
|
typeof handoff.confidence !== "number" ||
|
|
typeof handoff.expectedOutput !== "string"
|
|
) {
|
|
return null;
|
|
}
|
|
return {
|
|
confidence: handoff.confidence,
|
|
evidenceRefs: Array.isArray(handoff.evidenceRefs)
|
|
? handoff.evidenceRefs.filter((item): item is string => typeof item === "string")
|
|
: [],
|
|
expectedOutput: handoff.expectedOutput,
|
|
reason: handoff.reason,
|
|
risk:
|
|
handoff.risk === "low" || handoff.risk === "medium" || handoff.risk === "high"
|
|
? handoff.risk
|
|
: "medium",
|
|
targetAgent: handoff.targetAgent,
|
|
taskSpec: handoff.taskSpec,
|
|
...(handoff.userVisibleMessage
|
|
? { userVisibleMessage: handoff.userVisibleMessage }
|
|
: {}),
|
|
};
|
|
}
|