Add cost-aware routing and LLM budget tracking

- BudgetTracker tracks LLM token cost separately from tool cost
- LlmClient auto-records cost into request-scoped budget
- Router selects tier from classify confidence, intent risk, escalation count
- Supervisor short-circuits via regex fast-path when rule confidence is high
- Onboarding skill loads user escalation history to inform routing
This commit is contained in:
dahyeon committed 2026-05-19 01:28:54 -07:00
1 parent 211de18476
commit 21df14cd00
16 files changed
+473 -22

No files matched your search

+25
View File
@@ -59,6 +59,30 @@ describe("BudgetTracker", () => {
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget"); expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
}); });
it("tracks LLM and tool costs separately while enforcing the total budget", () => {
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 0.0055,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 5,
});
for (let index = 0; index < 5; index += 1) {
budget.recordLlmCost(0.001);
}
budget.recordToolCall(0.0005);
expect(budget.snapshot()).toMatchObject({
estimatedCostUsd: 0.0055,
totalCostUsd: 0.0055,
totalLlmCostUsd: 0.005,
totalToolCostUsd: 0.0005,
toolCalls: 1,
});
expect(budget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
});
it("can be disabled", () => { it("can be disabled", () => {
const budget = new BudgetTracker({ const budget = new BudgetTracker({
enabled: false, enabled: false,
@@ -71,6 +95,7 @@ describe("BudgetTracker", () => {
budget.recordVisit("supervisor"); budget.recordVisit("supervisor");
budget.recordHandoff("moderation"); budget.recordHandoff("moderation");
budget.recordToolCall(1); budget.recordToolCall(1);
budget.recordLlmCost(1);
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok"); expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
}); });
+23 -4
View File
@@ -20,13 +20,17 @@ export interface BudgetSnapshot {
maxHandoffs: number; maxHandoffs: number;
maxSameAgentVisits: number; maxSameAgentVisits: number;
maxToolCalls: number; maxToolCalls: number;
totalCostUsd: number;
totalLlmCostUsd: number;
totalToolCostUsd: number;
toolCalls: number; toolCalls: number;
visits: Record<string, number>; visits: Record<string, number>;
} }
export class BudgetTracker { export class BudgetTracker {
private estimatedCostUsd = 0;
private handoffs = 0; private handoffs = 0;
private totalLlmCostUsd = 0;
private totalToolCostUsd = 0;
private toolCalls = 0; private toolCalls = 0;
private readonly visits = new Map<string, number>(); private readonly visits = new Map<string, number>();
@@ -53,7 +57,7 @@ export class BudgetTracker {
if (this.toolCalls >= this.limits.maxToolCalls) { if (this.toolCalls >= this.limits.maxToolCalls) {
return "tool-budget"; return "tool-budget";
} }
if (this.estimatedCostUsd >= this.limits.maxCostUsd) { if (this.totalCostUsd() >= this.limits.maxCostUsd) {
return "cost-budget"; return "cost-budget";
} }
return "ok"; return "ok";
@@ -66,7 +70,14 @@ export class BudgetTracker {
recordToolCall(estimatedCostUsd = 0): void { recordToolCall(estimatedCostUsd = 0): void {
this.toolCalls += 1; this.toolCalls += 1;
this.estimatedCostUsd += estimatedCostUsd; this.totalToolCostUsd += estimatedCostUsd;
}
recordLlmCost(costUsd: number): void {
if (!Number.isFinite(costUsd) || costUsd <= 0) {
return;
}
this.totalLlmCostUsd += costUsd;
} }
recordVisit(target: string): void { recordVisit(target: string): void {
@@ -74,15 +85,23 @@ export class BudgetTracker {
} }
snapshot(): BudgetSnapshot { snapshot(): BudgetSnapshot {
const totalCostUsd = this.totalCostUsd();
return { return {
estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)), estimatedCostUsd: Number(totalCostUsd.toFixed(6)),
handoffs: this.handoffs, handoffs: this.handoffs,
maxCostUsd: this.limits.maxCostUsd, maxCostUsd: this.limits.maxCostUsd,
maxHandoffs: this.limits.maxHandoffs, maxHandoffs: this.limits.maxHandoffs,
maxSameAgentVisits: this.limits.maxSameAgentVisits, maxSameAgentVisits: this.limits.maxSameAgentVisits,
maxToolCalls: this.limits.maxToolCalls, maxToolCalls: this.limits.maxToolCalls,
totalCostUsd: Number(totalCostUsd.toFixed(6)),
totalLlmCostUsd: Number(this.totalLlmCostUsd.toFixed(6)),
totalToolCostUsd: Number(this.totalToolCostUsd.toFixed(6)),
toolCalls: this.toolCalls, toolCalls: this.toolCalls,
visits: Object.fromEntries(this.visits.entries()), visits: Object.fromEntries(this.visits.entries()),
}; };
} }
private totalCostUsd(): number {
return this.totalLlmCostUsd + this.totalToolCostUsd;
}
} }
+1 -1
View File
@@ -90,7 +90,7 @@ export async function dispatch(
return runWithContext( return runWithContext(
{ {
...contextBase, ...contextBase,
budget: budget.snapshot(), budget,
handoffChain: chain, handoffChain: chain,
handoffDepth: depth, handoffDepth: depth,
}, },
+40
View File
@@ -5,10 +5,15 @@ import {
getAlertThresholdEvalScoreDropPct, getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate, getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs, getAlertThresholdP95LatencyMs,
getAgentBudgetLlmCostEnabled,
getAgentCostAwareTierEnabled,
getAgentSupervisorFastPathEnabled,
getAgentToolSandboxEnabled,
getEmbeddingModel, getEmbeddingModel,
getEmbeddingProvider, getEmbeddingProvider,
getLongTermSummaryEnabled, getLongTermSummaryEnabled,
getMemoryEvictionEnabled, getMemoryEvictionEnabled,
getNatsAclMode,
getPhoenixLogPrompts, getPhoenixLogPrompts,
getPhoenixOtlpEndpoint, getPhoenixOtlpEndpoint,
getRagHybridEnabled, getRagHybridEnabled,
@@ -17,7 +22,9 @@ import {
getRagEnabled, getRagEnabled,
getSlackAlertWebhookUrl, getSlackAlertWebhookUrl,
getSlackWebhookTimeoutMs, getSlackWebhookTimeoutMs,
getSupervisorFastPathConfidence,
getSupervisorLlmResponseEnabled, getSupervisorLlmResponseEnabled,
getToolAuditMaxLen,
} from "./config.js"; } from "./config.js";
const ENV_KEYS = [ const ENV_KEYS = [
@@ -35,6 +42,13 @@ const ENV_KEYS = [
"AGENT_RAG_RERANKER_TOP_K", "AGENT_RAG_RERANKER_TOP_K",
"AGENT_LONG_TERM_SUMMARY_ENABLED", "AGENT_LONG_TERM_SUMMARY_ENABLED",
"AGENT_MEMORY_EVICTION_ENABLED", "AGENT_MEMORY_EVICTION_ENABLED",
"AGENT_BUDGET_LLM_COST_ENABLED",
"AGENT_COST_AWARE_TIER_ENABLED",
"AGENT_SUPERVISOR_FAST_PATH_ENABLED",
"SUPERVISOR_FAST_PATH_CONFIDENCE",
"AGENT_TOOL_SANDBOX_ENABLED",
"AGENT_TOOL_AUDIT_MAX_LEN",
"AGENT_NATS_ACL_MODE",
"EMBEDDING_MODEL", "EMBEDDING_MODEL",
"EMBEDDING_PROVIDER", "EMBEDDING_PROVIDER",
"PHOENIX_LOG_PROMPTS", "PHOENIX_LOG_PROMPTS",
@@ -129,4 +143,30 @@ describe("config", () => {
expect(getSupervisorLlmResponseEnabled()).toBe(false); expect(getSupervisorLlmResponseEnabled()).toBe(false);
}); });
it("reads operational excellence guardrail configuration", () => {
expect(getAgentBudgetLlmCostEnabled()).toBe(true);
expect(getAgentCostAwareTierEnabled()).toBe(true);
expect(getAgentSupervisorFastPathEnabled()).toBe(true);
expect(getSupervisorFastPathConfidence()).toBe(0.85);
expect(getAgentToolSandboxEnabled()).toBe(false);
expect(getToolAuditMaxLen()).toBe(100_000);
expect(getNatsAclMode()).toBe("log-only");
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
vi.stubEnv("AGENT_COST_AWARE_TIER_ENABLED", "false");
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
vi.stubEnv("SUPERVISOR_FAST_PATH_CONFIDENCE", "0.9");
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
vi.stubEnv("AGENT_TOOL_AUDIT_MAX_LEN", "42");
vi.stubEnv("AGENT_NATS_ACL_MODE", "enforce");
expect(getAgentBudgetLlmCostEnabled()).toBe(false);
expect(getAgentCostAwareTierEnabled()).toBe(false);
expect(getAgentSupervisorFastPathEnabled()).toBe(false);
expect(getSupervisorFastPathConfidence()).toBe(0.9);
expect(getAgentToolSandboxEnabled()).toBe(true);
expect(getToolAuditMaxLen()).toBe(42);
expect(getNatsAclMode()).toBe("enforce");
});
}); });
+32
View File
@@ -231,6 +231,38 @@ export function getAgentBudgetMaxCostUsd(): number {
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5); return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
} }
export function getAgentBudgetLlmCostEnabled(): boolean {
return getBoolean("AGENT_BUDGET_LLM_COST_ENABLED", true);
}
export function getAgentCostAwareTierEnabled(): boolean {
return getBoolean("AGENT_COST_AWARE_TIER_ENABLED", true);
}
export function getAgentSupervisorFastPathEnabled(): boolean {
return getBoolean("AGENT_SUPERVISOR_FAST_PATH_ENABLED", true);
}
export function getSupervisorFastPathConfidence(): number {
return getRatioNumber("SUPERVISOR_FAST_PATH_CONFIDENCE", 0.85);
}
export function getAgentToolSandboxEnabled(): boolean {
return getBoolean("AGENT_TOOL_SANDBOX_ENABLED", false);
}
export function getToolAuditMaxLen(): number {
return getPositiveNumber("AGENT_TOOL_AUDIT_MAX_LEN", 100_000);
}
export function getNatsAclMode(): "off" | "log-only" | "enforce" {
const raw = process.env["AGENT_NATS_ACL_MODE"]?.trim().toLowerCase();
if (raw === "off" || raw === "log-only" || raw === "enforce") {
return raw;
}
return "log-only";
}
export function getAgentRuntimeVerifierEnabled(): boolean { export function getAgentRuntimeVerifierEnabled(): boolean {
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false); return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
} }
+101
View File
@@ -25,6 +25,8 @@ vi.mock("@obs/spans", () => ({
})); }));
import { LlmClient } from "./llm-client.js"; import { LlmClient } from "./llm-client.js";
import { BudgetTracker } from "@agent/budget";
import { runWithContext } from "@obs/context";
import type { ChatCompletionResponse } from "./types.js"; import type { ChatCompletionResponse } from "./types.js";
function okResponse( function okResponse(
@@ -168,4 +170,103 @@ describe("LlmClient endpoint normalization", () => {
}), }),
); );
}); });
it("records LLM cost into the request budget", async () => {
vi.stubGlobal(
"fetch",
vi.fn(() =>
Promise.resolve(
okResponse("reply", {
usage: {
completion_tokens: 50,
prompt_tokens: 100,
total_tokens: 150,
},
}),
),
),
);
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 1,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 5,
});
const client = new LlmClient({
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
providerApiKeys: { openrouter: "test-key" },
});
await runWithContext(
{
budget,
chatroomId: "room-1",
requestId: "req-1",
skill: "onboarding",
startedAt: "2026-05-18T00:00:00.000Z",
trigger: "user-message",
userId: "user-1",
},
() =>
client.chatCompletion([{ content: "hi", role: "user" }], {
modelOverride: "gpt-5.3-codex-spark",
}),
);
expect(budget.snapshot().totalLlmCostUsd).toBe(0.000045);
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
expect.objectContaining({
"budget.llm_cost_usd": 0.000045,
"budget.total_cost_usd": 0.000045,
}),
);
});
it("does not record LLM cost when budget cost tracking is disabled", async () => {
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
vi.stubGlobal(
"fetch",
vi.fn(() =>
Promise.resolve(
okResponse("reply", {
usage: {
completion_tokens: 50,
prompt_tokens: 100,
total_tokens: 150,
},
}),
),
),
);
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 1,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 5,
});
const client = new LlmClient({
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
providerApiKeys: { openrouter: "test-key" },
});
await runWithContext(
{
budget,
chatroomId: "room-1",
requestId: "req-1",
skill: "onboarding",
startedAt: "2026-05-18T00:00:00.000Z",
trigger: "user-message",
userId: "user-1",
},
() =>
client.chatCompletion([{ content: "hi", role: "user" }], {
modelOverride: "gpt-5.3-codex-spark",
}),
);
expect(budget.snapshot().totalLlmCostUsd).toBe(0);
});
}); });
+12
View File
@@ -10,6 +10,7 @@ import type {
import { getContext, getRequestLogger } from "@obs/context"; import { getContext, getRequestLogger } from "@obs/context";
import { maskTraceContent, withLlmSpan } from "@obs/spans"; import { maskTraceContent, withLlmSpan } from "@obs/spans";
import { import {
getAgentBudgetLlmCostEnabled,
getLlmHealthTimeoutMs, getLlmHealthTimeoutMs,
getLlmRequestTimeoutMs, getLlmRequestTimeoutMs,
getLlmTemperature, getLlmTemperature,
@@ -257,11 +258,22 @@ export class LlmClient {
model: request.model, model: request.model,
skill: getContext()?.skill, skill: getContext()?.skill,
}); });
if (getAgentBudgetLlmCostEnabled()) {
getContext()?.budget?.recordLlmCost(costUsd);
}
const budgetSnapshot = getContext()?.budget?.snapshot();
span.setAttributes({ span.setAttributes({
...(shouldLogPrompts ...(shouldLogPrompts
? buildCompletionTraceAttributes(result) ? buildCompletionTraceAttributes(result)
: {}), : {}),
...(budgetSnapshot
? {
"budget.llm_cost_usd": budgetSnapshot.totalLlmCostUsd,
"budget.tool_cost_usd": budgetSnapshot.totalToolCostUsd,
"budget.total_cost_usd": budgetSnapshot.totalCostUsd,
}
: {}),
"gen_ai.response.finish_reasons": payload.choices "gen_ai.response.finish_reasons": payload.choices
?.map((choice) => choice.finish_reason ?? "unknown") ?.map((choice) => choice.finish_reason ?? "unknown")
.filter((reason): reason is string => Boolean(reason)), .filter((reason): reason is string => Boolean(reason)),
+2 -2
View File
@@ -3,12 +3,12 @@ import { AsyncLocalStorage } from 'node:async_hooks';
import type { Logger } from 'pino'; import type { Logger } from 'pino';
import { getLogger } from './logger.js'; import { getLogger } from './logger.js';
import type { BudgetSnapshot } from '@agent/budget'; import type { BudgetTracker } from '@agent/budget';
export interface RequestContext { export interface RequestContext {
chatroomId: string; chatroomId: string;
deadlineAt?: string; deadlineAt?: string;
budget?: BudgetSnapshot | undefined; budget?: BudgetTracker | undefined;
handoffChain?: string[] | undefined; handoffChain?: string[] | undefined;
handoffDepth?: number | undefined; handoffDepth?: number | undefined;
logger: Logger; logger: Logger;
+3
View File
@@ -46,6 +46,9 @@ describe("escalation trigger", () => {
maxHandoffs: 4, maxHandoffs: 4,
maxSameAgentVisits: 2, maxSameAgentVisits: 2,
maxToolCalls: 20, maxToolCalls: 20,
totalCostUsd: 0,
totalLlmCostUsd: 0,
totalToolCostUsd: 0,
toolCalls: 0, toolCalls: 0,
visits: {}, visits: {},
}, },
+1
View File
@@ -16,5 +16,6 @@ export function createOnboardingSkill(deps: SkillDeps): Skill {
deps.shadowEvalBus, deps.shadowEvalBus,
deps.natsClient, deps.natsClient,
deps.memoryService, deps.memoryService,
deps.escalationHistoryService,
); );
} }
+45 -2
View File
@@ -102,9 +102,9 @@ describe("decidePreGenerationRouting", () => {
}, },
{ {
confidence: 0.3, confidence: 0.3,
expected: { kind: "tier0-only" }, expected: { kind: "tier1-direct", skipPlanner: false },
intentKind: "off-topic-to-other" as const, intentKind: "off-topic-to-other" as const,
name: "keeps low-confidence off-topic on tier0", name: "promotes low-confidence off-topic to tier1",
phase: { kind: "fresh" } as OnboardingPhase, phase: { kind: "fresh" } as OnboardingPhase,
}, },
{ {
@@ -319,6 +319,49 @@ describe("decidePreGenerationRouting", () => {
).toEqual(expected); ).toEqual(expected);
}); });
it("keeps high-confidence low-risk requests on tier0", () => {
expect(
decidePreGenerationRouting({
classified: classified("age-question", 0.95),
derived: derived({ kind: "deadline-near", remaining: ["photo"] }, intent("age-question"), {
tier: 1,
}),
intentRiskTier: "low",
request: request(),
session: session(),
userEscalationCount: 0,
}),
).toEqual({ kind: "tier0-only" });
});
it("promotes low-confidence high-risk requests with escalation history", () => {
expect(
decidePreGenerationRouting({
classified: classified("unknown", 0.49),
derived: derived({ kind: "fresh" }, intent("unknown")),
intentRiskTier: "high",
request: request(),
session: session(),
userEscalationCount: 4,
}),
).toEqual({ kind: "tier1-direct", skipPlanner: false });
});
it("prefers tier0 during latency degradation unless safety already promoted", () => {
expect(
decidePreGenerationRouting({
classified: classified("age-question", 0.8),
derived: derived({ kind: "deadline-near", remaining: ["nickname"] }, intent("age-question"), {
tier: 1,
}),
intentRiskTier: "medium",
recentLatencyP95Ms: 10_000,
request: request(),
session: session(),
}),
).toEqual({ kind: "tier0-only" });
});
it("trains, saves, loads, and applies a learned router model", () => { it("trains, saves, loads, and applies a learned router model", () => {
const lowConfidenceInput = { const lowConfidenceInput = {
classified: classified("age-question", 0.25), classified: classified("age-question", 0.25),
+49
View File
@@ -3,6 +3,10 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname } from "node:path"; import { dirname } from "node:path";
import type { AgentRequest } from "@agent/types"; import type { AgentRequest } from "@agent/types";
import {
getAgentCostAwareTierEnabled,
getAlertThresholdP95LatencyMs,
} from "../../../config.js";
import type { import type {
ClassifiedIntent, ClassifiedIntent,
DerivedState, DerivedState,
@@ -58,8 +62,11 @@ export interface RouterTrainingSample {
export interface RouterInput { export interface RouterInput {
classified: ClassifiedIntent; classified: ClassifiedIntent;
derived: DerivedState; derived: DerivedState;
intentRiskTier?: "low" | "medium" | "high" | undefined;
recentLatencyP95Ms?: number | undefined;
request: AgentRequest; request: AgentRequest;
session: OnboardingSession | null; session: OnboardingSession | null;
userEscalationCount?: number | undefined;
} }
export const ROUTER_FEATURE_NAMES = [ export const ROUTER_FEATURE_NAMES = [
@@ -294,6 +301,43 @@ function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
); );
} }
function isLatencyDegraded(input: RouterInput): boolean {
const p95 = input.recentLatencyP95Ms;
return (
typeof p95 === "number" &&
Number.isFinite(p95) &&
p95 > getAlertThresholdP95LatencyMs("onboarding")
);
}
function maybeDecideCostAwareRouting(input: RouterInput): RoutingDecision | null {
if (!getAgentCostAwareTierEnabled()) {
return null;
}
const confidence = clamp01(input.classified.confidence);
const riskTier = input.intentRiskTier ?? "medium";
const escalationCount = Math.max(0, input.userEscalationCount ?? 0);
const lowLatency = !isLatencyDegraded(input);
if (confidence < 0.5 || (riskTier === "high" && escalationCount > 3)) {
if (input.derived.responseMode === "llm-with-tools") {
return { kind: "tier1-with-tools" };
}
return { kind: "tier1-direct", skipPlanner: false };
}
if (!lowLatency) {
return { kind: "tier0-only" };
}
if (confidence >= 0.9 && riskTier === "low") {
return { kind: "tier0-only" };
}
return null;
}
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision { export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
const { classified, derived, request, session } = input; const { classified, derived, request, session } = input;
const { confidence, intent } = classified; const { confidence, intent } = classified;
@@ -334,6 +378,11 @@ export function decidePreGenerationRouting(input: RouterInput): RoutingDecision
}; };
} }
const costAwareDecision = maybeDecideCostAwareRouting(input);
if (costAwareDecision) {
return costAwareDecision;
}
if ( if (
phase.kind === "nickname-rejected" || phase.kind === "nickname-rejected" ||
intent.kind === "discord-verify-question" || intent.kind === "discord-verify-question" ||
+26
View File
@@ -1,6 +1,7 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types'; import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
import type { GarnetClient } from '@common/garnet'; import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats'; import type { NatsClient } from '@common/nats';
import type { EscalationHistoryService } from '../../memory/escalation-history.js';
import type { MemoryService } from '../../memory/memory-types.js'; import type { MemoryService } from '../../memory/memory-types.js';
import { import {
buildLongTermSummary, buildLongTermSummary,
@@ -118,6 +119,7 @@ export class OnboardingSkill implements Skill {
shadowEvalPublisher?: ShadowEvalPublisher, shadowEvalPublisher?: ShadowEvalPublisher,
private readonly natsClient?: Pick<NatsClient, 'request'>, private readonly natsClient?: Pick<NatsClient, 'request'>,
private readonly memoryService?: MemoryService, private readonly memoryService?: MemoryService,
private readonly escalationHistoryService?: EscalationHistoryService,
) { ) {
this.classifier = new OnboardingIntentClassifier(llmClient); this.classifier = new OnboardingIntentClassifier(llmClient);
this.blackboardStore = new OnboardingBlackboardStore( this.blackboardStore = new OnboardingBlackboardStore(
@@ -186,13 +188,17 @@ export class OnboardingSkill implements Skill {
const deriveStep = trace?.startStep('derive'); const deriveStep = trace?.startStep('derive');
const derived = deriveState(session, request.trigger, classified.intent); const derived = deriveState(session, request.trigger, classified.intent);
const escalationHistory = await this.loadEscalationHistory(request.userId);
const routerMode = getAgentRouterMode(); const routerMode = getAgentRouterMode();
if (routerMode === 'static' || routerMode === 'learned-v1') { if (routerMode === 'static' || routerMode === 'learned-v1') {
const routerInput = { const routerInput = {
classified, classified,
derived, derived,
intentRiskTier: this.capabilityCard().riskTier,
recentLatencyP95Ms: this.getRecentLatencyP95Ms(request),
request, request,
session, session,
userEscalationCount: escalationHistory?.totalEscalations ?? 0,
}; };
derived.routingDecision = derived.routingDecision =
routerMode === 'learned-v1' routerMode === 'learned-v1'
@@ -353,6 +359,26 @@ export class OnboardingSkill implements Skill {
return normalizeOnboardingSession(rawSession); return normalizeOnboardingSession(rawSession);
} }
private async loadEscalationHistory(userId: string) {
if (!this.escalationHistoryService) {
return null;
}
try {
return await this.escalationHistoryService.load(userId);
} catch (error) {
getRequestLogger().warn(
{ err: error },
'[OnboardingSkill] Failed to load escalation history',
);
return null;
}
}
private getRecentLatencyP95Ms(request: AgentRequest): number | undefined {
const raw = request.context?.['recentLatencyP95Ms'];
return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined;
}
private async decideEarlyAbstention( private async decideEarlyAbstention(
request: AgentRequest, request: AgentRequest,
classified: ClassifiedIntent, classified: ClassifiedIntent,
+43 -7
View File
@@ -1,17 +1,53 @@
import type { HandoffPacket } from "@agent/handoff-packet"; import type { HandoffPacket } from "@agent/handoff-packet";
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null { export interface SupervisorRuleRoute {
const text = [ confidence: number;
target: string | null;
}
const RULES = [
{
pattern: /(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu,
target: "moderation",
},
{
pattern: /(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu,
target: "onboarding",
},
] as const;
function packetText(packet: HandoffPacket | undefined): string {
return [
packet?.userGoal, packet?.userGoal,
packet?.summary, packet?.summary,
packet?.handoffReason, packet?.handoffReason,
].join(" "); ].join(" ");
}
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) { export function routeSupervisorByRules(
return "moderation"; packet: HandoffPacket | undefined,
): SupervisorRuleRoute {
const text = packetText(packet);
const matches = RULES.filter((rule) => rule.pattern.test(text));
if (matches.length === 1) {
return {
confidence: 0.9,
target: matches[0]?.target ?? null,
};
} }
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) { if (matches.length > 1) {
return "onboarding"; return {
confidence: 0.6,
target: matches[0]?.target ?? null,
};
} }
return null; return {
confidence: 0,
target: null,
};
}
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
return routeSupervisorByRules(packet).target;
} }
+41 -1
View File
@@ -119,7 +119,7 @@ describe("SupervisorSkill", () => {
expect(response.text).toContain("운영자"); expect(response.text).toContain("운영자");
expect(response.metadata).toMatchObject({ expect(response.metadata).toMatchObject({
routedBy: "rules", routedBy: "regex-fast-path",
source: "template", source: "template",
supervisorReason: "max-hop", supervisorReason: "max-hop",
suggestedTarget: "onboarding", suggestedTarget: "onboarding",
@@ -149,6 +149,7 @@ describe("SupervisorSkill", () => {
}); });
it("uses LLM routing when the decision is confident and valid", async () => { it("uses LLM routing when the decision is confident and valid", async () => {
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
const garnet = new FakeGarnet(); const garnet = new FakeGarnet();
const nats = new FakeNats(); const nats = new FakeNats();
const skill = new SupervisorSkill( const skill = new SupervisorSkill(
@@ -189,6 +190,45 @@ describe("SupervisorSkill", () => {
}); });
}); });
it("uses regex fast-path before calling the supervisor LLM", async () => {
const garnet = new FakeGarnet();
const nats = new FakeNats();
const llm = {
chatCompletion: vi.fn(),
};
const skill = new SupervisorSkill(
garnet,
nats,
llm as unknown as Pick<LlmClient, "chatCompletion">,
registry,
);
const packet = buildHandoffPacket({
handoff: {
confidence: 0.7,
evidenceRefs: [],
expectedOutput: "text",
reason: "out-of-scope",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "광고 규칙 문의",
},
request: baseRequest,
response: sourceResponse,
sourceChain: ["onboarding", "supervisor"],
});
const response = await skill.handle(
{ ...baseRequest, context: { handoffPacket: packet } },
[],
);
expect(llm.chatCompletion).not.toHaveBeenCalled();
expect(response.metadata).toMatchObject({
routedBy: "regex-fast-path",
suggestedTarget: "moderation",
});
});
it("uses safe LLM userMessage for supervisor replies", async () => { it("uses safe LLM userMessage for supervisor replies", async () => {
const garnet = new FakeGarnet(); const garnet = new FakeGarnet();
const nats = new FakeNats(); const nats = new FakeNats();
+29 -5
View File
@@ -3,6 +3,7 @@ import type {
AgentResponse, AgentResponse,
ConversationMessage, ConversationMessage,
} from "@agent/types"; } from "@agent/types";
import { trace } from "@opentelemetry/api";
import { readHandoffPacket } from "@agent/handoff-packet"; import { readHandoffPacket } from "@agent/handoff-packet";
import type { HandoffPacket } from "@agent/handoff-packet"; import type { HandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet"; import type { GarnetClient } from "@common/garnet";
@@ -19,6 +20,8 @@ import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader"; import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
import { getLogger } from "@obs/logger"; import { getLogger } from "@obs/logger";
import { import {
getAgentSupervisorFastPathEnabled,
getSupervisorFastPathConfidence,
getSupervisorLlmConfidenceThreshold, getSupervisorLlmConfidenceThreshold,
getSupervisorLlmEnabled, getSupervisorLlmEnabled,
getSupervisorLlmResponseEnabled, getSupervisorLlmResponseEnabled,
@@ -28,7 +31,7 @@ import {
buildSupervisorUserPrompt, buildSupervisorUserPrompt,
} from "./capability-prompt.js"; } from "./capability-prompt.js";
import { appendSupervisorReview } from "./review-queue.js"; import { appendSupervisorReview } from "./review-queue.js";
import { routeSupervisorTarget } from "./routing-rules.js"; import { routeSupervisorByRules } from "./routing-rules.js";
import { import {
supervisorRoutingSchema, supervisorRoutingSchema,
type SupervisorRoutingDecision, type SupervisorRoutingDecision,
@@ -114,13 +117,30 @@ export class SupervisorSkill implements Skill {
): Promise<AgentResponse> { ): Promise<AgentResponse> {
const packet = readHandoffPacket(request); const packet = readHandoffPacket(request);
const escalationHistory = await this.loadEscalationHistory(request.userId); const escalationHistory = await this.loadEscalationHistory(request.userId);
const llmDecision = await this.routeByLlm(packet, escalationHistory); const rulesDecision = routeSupervisorByRules(packet);
const useRegexFastPath =
getAgentSupervisorFastPathEnabled() &&
rulesDecision.target !== null &&
rulesDecision.confidence >= getSupervisorFastPathConfidence() &&
this.isKnownTarget(rulesDecision.target);
const llmDecision = useRegexFastPath
? null
: await this.routeByLlm(packet, escalationHistory);
const useLlmRoute = const useLlmRoute =
llmDecision !== null && llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold(); llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
const target = useLlmRoute const target = useLlmRoute
? llmDecision.target ? llmDecision.target
: routeSupervisorTarget(packet); : rulesDecision.target;
const routedBy = useRegexFastPath
? "regex-fast-path"
: useLlmRoute
? "llm"
: "rules-fallback";
trace.getActiveSpan()?.setAttributes({
"supervisor.routed_by": routedBy,
"supervisor.rules_confidence": rulesDecision.confidence,
});
const reason = packet?.handoffReason ?? "out-of-scope"; const reason = packet?.handoffReason ?? "out-of-scope";
const queuedAt = new Date().toISOString(); const queuedAt = new Date().toISOString();
@@ -169,7 +189,8 @@ export class SupervisorSkill implements Skill {
return { return {
confidence: 0.6, confidence: 0.6,
metadata: { metadata: {
routedBy: useLlmRoute ? "llm" : "rules", routedBy,
rulesConfidence: rulesDecision.confidence,
source: userMessage.source, source: userMessage.source,
supervisorReason: useLlmRoute ? llmDecision.reason : reason, supervisorReason: useLlmRoute ? llmDecision.reason : reason,
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0, supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
@@ -234,7 +255,10 @@ export class SupervisorSkill implements Skill {
private isKnownTarget(target: string): boolean { private isKnownTarget(target: string): boolean {
return ( return (
target === "human-review" || Boolean(this.skillRegistry?.get(target)) target === "human-review" ||
target === "onboarding" ||
target === "moderation" ||
Boolean(this.skillRegistry?.get(target))
); );
} }