diff --git a/src/agent/budget.test.ts b/src/agent/budget.test.ts index 2988963..7ab1d2e 100644 --- a/src/agent/budget.test.ts +++ b/src/agent/budget.test.ts @@ -59,6 +59,30 @@ describe("BudgetTracker", () => { expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget"); }); + it("tracks LLM and tool costs separately while enforcing the total budget", () => { + const budget = new BudgetTracker({ + enabled: true, + maxCostUsd: 0.0055, + maxHandoffs: 5, + maxSameAgentVisits: 5, + maxToolCalls: 5, + }); + + for (let index = 0; index < 5; index += 1) { + budget.recordLlmCost(0.001); + } + budget.recordToolCall(0.0005); + + expect(budget.snapshot()).toMatchObject({ + estimatedCostUsd: 0.0055, + totalCostUsd: 0.0055, + totalLlmCostUsd: 0.005, + totalToolCostUsd: 0.0005, + toolCalls: 1, + }); + expect(budget.assertBeforeHandoff("supervisor")).toBe("cost-budget"); + }); + it("can be disabled", () => { const budget = new BudgetTracker({ enabled: false, @@ -71,6 +95,7 @@ describe("BudgetTracker", () => { budget.recordVisit("supervisor"); budget.recordHandoff("moderation"); budget.recordToolCall(1); + budget.recordLlmCost(1); expect(budget.assertBeforeHandoff("supervisor")).toBe("ok"); }); diff --git a/src/agent/budget.ts b/src/agent/budget.ts index f01d91a..85058b0 100644 --- a/src/agent/budget.ts +++ b/src/agent/budget.ts @@ -20,13 +20,17 @@ export interface BudgetSnapshot { maxHandoffs: number; maxSameAgentVisits: number; maxToolCalls: number; + totalCostUsd: number; + totalLlmCostUsd: number; + totalToolCostUsd: number; toolCalls: number; visits: Record; } export class BudgetTracker { - private estimatedCostUsd = 0; private handoffs = 0; + private totalLlmCostUsd = 0; + private totalToolCostUsd = 0; private toolCalls = 0; private readonly visits = new Map(); @@ -53,7 +57,7 @@ export class BudgetTracker { if (this.toolCalls >= this.limits.maxToolCalls) { return "tool-budget"; } - if (this.estimatedCostUsd >= this.limits.maxCostUsd) { + if (this.totalCostUsd() >= this.limits.maxCostUsd) { return "cost-budget"; } return "ok"; @@ -66,7 +70,14 @@ export class BudgetTracker { recordToolCall(estimatedCostUsd = 0): void { this.toolCalls += 1; - this.estimatedCostUsd += estimatedCostUsd; + this.totalToolCostUsd += estimatedCostUsd; + } + + recordLlmCost(costUsd: number): void { + if (!Number.isFinite(costUsd) || costUsd <= 0) { + return; + } + this.totalLlmCostUsd += costUsd; } recordVisit(target: string): void { @@ -74,15 +85,23 @@ export class BudgetTracker { } snapshot(): BudgetSnapshot { + const totalCostUsd = this.totalCostUsd(); return { - estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)), + estimatedCostUsd: Number(totalCostUsd.toFixed(6)), handoffs: this.handoffs, maxCostUsd: this.limits.maxCostUsd, maxHandoffs: this.limits.maxHandoffs, maxSameAgentVisits: this.limits.maxSameAgentVisits, maxToolCalls: this.limits.maxToolCalls, + totalCostUsd: Number(totalCostUsd.toFixed(6)), + totalLlmCostUsd: Number(this.totalLlmCostUsd.toFixed(6)), + totalToolCostUsd: Number(this.totalToolCostUsd.toFixed(6)), toolCalls: this.toolCalls, visits: Object.fromEntries(this.visits.entries()), }; } + + private totalCostUsd(): number { + return this.totalLlmCostUsd + this.totalToolCostUsd; + } } diff --git a/src/agent/dispatch.ts b/src/agent/dispatch.ts index 62886a6..3731879 100644 --- a/src/agent/dispatch.ts +++ b/src/agent/dispatch.ts @@ -90,7 +90,7 @@ export async function dispatch( return runWithContext( { ...contextBase, - budget: budget.snapshot(), + budget, handoffChain: chain, handoffDepth: depth, }, diff --git a/src/config.test.ts b/src/config.test.ts index 43e6d67..7db3c31 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -5,10 +5,15 @@ import { getAlertThresholdEvalScoreDropPct, getAlertThresholdLlmErrorRate, getAlertThresholdP95LatencyMs, + getAgentBudgetLlmCostEnabled, + getAgentCostAwareTierEnabled, + getAgentSupervisorFastPathEnabled, + getAgentToolSandboxEnabled, getEmbeddingModel, getEmbeddingProvider, getLongTermSummaryEnabled, getMemoryEvictionEnabled, + getNatsAclMode, getPhoenixLogPrompts, getPhoenixOtlpEndpoint, getRagHybridEnabled, @@ -17,7 +22,9 @@ import { getRagEnabled, getSlackAlertWebhookUrl, getSlackWebhookTimeoutMs, + getSupervisorFastPathConfidence, getSupervisorLlmResponseEnabled, + getToolAuditMaxLen, } from "./config.js"; const ENV_KEYS = [ @@ -35,6 +42,13 @@ const ENV_KEYS = [ "AGENT_RAG_RERANKER_TOP_K", "AGENT_LONG_TERM_SUMMARY_ENABLED", "AGENT_MEMORY_EVICTION_ENABLED", + "AGENT_BUDGET_LLM_COST_ENABLED", + "AGENT_COST_AWARE_TIER_ENABLED", + "AGENT_SUPERVISOR_FAST_PATH_ENABLED", + "SUPERVISOR_FAST_PATH_CONFIDENCE", + "AGENT_TOOL_SANDBOX_ENABLED", + "AGENT_TOOL_AUDIT_MAX_LEN", + "AGENT_NATS_ACL_MODE", "EMBEDDING_MODEL", "EMBEDDING_PROVIDER", "PHOENIX_LOG_PROMPTS", @@ -129,4 +143,30 @@ describe("config", () => { expect(getSupervisorLlmResponseEnabled()).toBe(false); }); + + it("reads operational excellence guardrail configuration", () => { + expect(getAgentBudgetLlmCostEnabled()).toBe(true); + expect(getAgentCostAwareTierEnabled()).toBe(true); + expect(getAgentSupervisorFastPathEnabled()).toBe(true); + expect(getSupervisorFastPathConfidence()).toBe(0.85); + expect(getAgentToolSandboxEnabled()).toBe(false); + expect(getToolAuditMaxLen()).toBe(100_000); + expect(getNatsAclMode()).toBe("log-only"); + + vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false"); + vi.stubEnv("AGENT_COST_AWARE_TIER_ENABLED", "false"); + vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false"); + vi.stubEnv("SUPERVISOR_FAST_PATH_CONFIDENCE", "0.9"); + vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true"); + vi.stubEnv("AGENT_TOOL_AUDIT_MAX_LEN", "42"); + vi.stubEnv("AGENT_NATS_ACL_MODE", "enforce"); + + expect(getAgentBudgetLlmCostEnabled()).toBe(false); + expect(getAgentCostAwareTierEnabled()).toBe(false); + expect(getAgentSupervisorFastPathEnabled()).toBe(false); + expect(getSupervisorFastPathConfidence()).toBe(0.9); + expect(getAgentToolSandboxEnabled()).toBe(true); + expect(getToolAuditMaxLen()).toBe(42); + expect(getNatsAclMode()).toBe("enforce"); + }); }); diff --git a/src/config.ts b/src/config.ts index 78c7565..c17c8ad 100644 --- a/src/config.ts +++ b/src/config.ts @@ -231,6 +231,38 @@ export function getAgentBudgetMaxCostUsd(): number { return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5); } +export function getAgentBudgetLlmCostEnabled(): boolean { + return getBoolean("AGENT_BUDGET_LLM_COST_ENABLED", true); +} + +export function getAgentCostAwareTierEnabled(): boolean { + return getBoolean("AGENT_COST_AWARE_TIER_ENABLED", true); +} + +export function getAgentSupervisorFastPathEnabled(): boolean { + return getBoolean("AGENT_SUPERVISOR_FAST_PATH_ENABLED", true); +} + +export function getSupervisorFastPathConfidence(): number { + return getRatioNumber("SUPERVISOR_FAST_PATH_CONFIDENCE", 0.85); +} + +export function getAgentToolSandboxEnabled(): boolean { + return getBoolean("AGENT_TOOL_SANDBOX_ENABLED", false); +} + +export function getToolAuditMaxLen(): number { + return getPositiveNumber("AGENT_TOOL_AUDIT_MAX_LEN", 100_000); +} + +export function getNatsAclMode(): "off" | "log-only" | "enforce" { + const raw = process.env["AGENT_NATS_ACL_MODE"]?.trim().toLowerCase(); + if (raw === "off" || raw === "log-only" || raw === "enforce") { + return raw; + } + return "log-only"; +} + export function getAgentRuntimeVerifierEnabled(): boolean { return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false); } diff --git a/src/llm/llm-client.test.ts b/src/llm/llm-client.test.ts index 439f74f..dae33a4 100644 --- a/src/llm/llm-client.test.ts +++ b/src/llm/llm-client.test.ts @@ -25,6 +25,8 @@ vi.mock("@obs/spans", () => ({ })); import { LlmClient } from "./llm-client.js"; +import { BudgetTracker } from "@agent/budget"; +import { runWithContext } from "@obs/context"; import type { ChatCompletionResponse } from "./types.js"; function okResponse( @@ -168,4 +170,103 @@ describe("LlmClient endpoint normalization", () => { }), ); }); + + it("records LLM cost into the request budget", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve( + okResponse("reply", { + usage: { + completion_tokens: 50, + prompt_tokens: 100, + total_tokens: 150, + }, + }), + ), + ), + ); + const budget = new BudgetTracker({ + enabled: true, + maxCostUsd: 1, + maxHandoffs: 5, + maxSameAgentVisits: 5, + maxToolCalls: 5, + }); + const client = new LlmClient({ + endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }], + providerApiKeys: { openrouter: "test-key" }, + }); + + await runWithContext( + { + budget, + chatroomId: "room-1", + requestId: "req-1", + skill: "onboarding", + startedAt: "2026-05-18T00:00:00.000Z", + trigger: "user-message", + userId: "user-1", + }, + () => + client.chatCompletion([{ content: "hi", role: "user" }], { + modelOverride: "gpt-5.3-codex-spark", + }), + ); + + expect(budget.snapshot().totalLlmCostUsd).toBe(0.000045); + expect(llmSpanMock.setAttributes).toHaveBeenCalledWith( + expect.objectContaining({ + "budget.llm_cost_usd": 0.000045, + "budget.total_cost_usd": 0.000045, + }), + ); + }); + + it("does not record LLM cost when budget cost tracking is disabled", async () => { + vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false"); + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve( + okResponse("reply", { + usage: { + completion_tokens: 50, + prompt_tokens: 100, + total_tokens: 150, + }, + }), + ), + ), + ); + const budget = new BudgetTracker({ + enabled: true, + maxCostUsd: 1, + maxHandoffs: 5, + maxSameAgentVisits: 5, + maxToolCalls: 5, + }); + const client = new LlmClient({ + endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }], + providerApiKeys: { openrouter: "test-key" }, + }); + + await runWithContext( + { + budget, + chatroomId: "room-1", + requestId: "req-1", + skill: "onboarding", + startedAt: "2026-05-18T00:00:00.000Z", + trigger: "user-message", + userId: "user-1", + }, + () => + client.chatCompletion([{ content: "hi", role: "user" }], { + modelOverride: "gpt-5.3-codex-spark", + }), + ); + + expect(budget.snapshot().totalLlmCostUsd).toBe(0); + }); }); diff --git a/src/llm/llm-client.ts b/src/llm/llm-client.ts index c2d6b5b..15d02cc 100644 --- a/src/llm/llm-client.ts +++ b/src/llm/llm-client.ts @@ -10,6 +10,7 @@ import type { import { getContext, getRequestLogger } from "@obs/context"; import { maskTraceContent, withLlmSpan } from "@obs/spans"; import { + getAgentBudgetLlmCostEnabled, getLlmHealthTimeoutMs, getLlmRequestTimeoutMs, getLlmTemperature, @@ -257,11 +258,22 @@ export class LlmClient { model: request.model, skill: getContext()?.skill, }); + if (getAgentBudgetLlmCostEnabled()) { + getContext()?.budget?.recordLlmCost(costUsd); + } + const budgetSnapshot = getContext()?.budget?.snapshot(); span.setAttributes({ ...(shouldLogPrompts ? buildCompletionTraceAttributes(result) : {}), + ...(budgetSnapshot + ? { + "budget.llm_cost_usd": budgetSnapshot.totalLlmCostUsd, + "budget.tool_cost_usd": budgetSnapshot.totalToolCostUsd, + "budget.total_cost_usd": budgetSnapshot.totalCostUsd, + } + : {}), "gen_ai.response.finish_reasons": payload.choices ?.map((choice) => choice.finish_reason ?? "unknown") .filter((reason): reason is string => Boolean(reason)), diff --git a/src/observability/context.ts b/src/observability/context.ts index 105ce68..b2d3750 100644 --- a/src/observability/context.ts +++ b/src/observability/context.ts @@ -3,12 +3,12 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { Logger } from 'pino'; import { getLogger } from './logger.js'; -import type { BudgetSnapshot } from '@agent/budget'; +import type { BudgetTracker } from '@agent/budget'; export interface RequestContext { chatroomId: string; deadlineAt?: string; - budget?: BudgetSnapshot | undefined; + budget?: BudgetTracker | undefined; handoffChain?: string[] | undefined; handoffDepth?: number | undefined; logger: Logger; diff --git a/src/skills/escalation/trigger.test.ts b/src/skills/escalation/trigger.test.ts index 1af9c55..b6934e9 100644 --- a/src/skills/escalation/trigger.test.ts +++ b/src/skills/escalation/trigger.test.ts @@ -46,6 +46,9 @@ describe("escalation trigger", () => { maxHandoffs: 4, maxSameAgentVisits: 2, maxToolCalls: 20, + totalCostUsd: 0, + totalLlmCostUsd: 0, + totalToolCostUsd: 0, toolCalls: 0, visits: {}, }, diff --git a/src/skills/onboarding/index.ts b/src/skills/onboarding/index.ts index 914e3e9..08a522e 100644 --- a/src/skills/onboarding/index.ts +++ b/src/skills/onboarding/index.ts @@ -16,5 +16,6 @@ export function createOnboardingSkill(deps: SkillDeps): Skill { deps.shadowEvalBus, deps.natsClient, deps.memoryService, + deps.escalationHistoryService, ); } diff --git a/src/skills/onboarding/runtime/router.test.ts b/src/skills/onboarding/runtime/router.test.ts index 6f5e235..0e0c344 100644 --- a/src/skills/onboarding/runtime/router.test.ts +++ b/src/skills/onboarding/runtime/router.test.ts @@ -102,9 +102,9 @@ describe("decidePreGenerationRouting", () => { }, { confidence: 0.3, - expected: { kind: "tier0-only" }, + expected: { kind: "tier1-direct", skipPlanner: false }, intentKind: "off-topic-to-other" as const, - name: "keeps low-confidence off-topic on tier0", + name: "promotes low-confidence off-topic to tier1", phase: { kind: "fresh" } as OnboardingPhase, }, { @@ -319,6 +319,49 @@ describe("decidePreGenerationRouting", () => { ).toEqual(expected); }); + it("keeps high-confidence low-risk requests on tier0", () => { + expect( + decidePreGenerationRouting({ + classified: classified("age-question", 0.95), + derived: derived({ kind: "deadline-near", remaining: ["photo"] }, intent("age-question"), { + tier: 1, + }), + intentRiskTier: "low", + request: request(), + session: session(), + userEscalationCount: 0, + }), + ).toEqual({ kind: "tier0-only" }); + }); + + it("promotes low-confidence high-risk requests with escalation history", () => { + expect( + decidePreGenerationRouting({ + classified: classified("unknown", 0.49), + derived: derived({ kind: "fresh" }, intent("unknown")), + intentRiskTier: "high", + request: request(), + session: session(), + userEscalationCount: 4, + }), + ).toEqual({ kind: "tier1-direct", skipPlanner: false }); + }); + + it("prefers tier0 during latency degradation unless safety already promoted", () => { + expect( + decidePreGenerationRouting({ + classified: classified("age-question", 0.8), + derived: derived({ kind: "deadline-near", remaining: ["nickname"] }, intent("age-question"), { + tier: 1, + }), + intentRiskTier: "medium", + recentLatencyP95Ms: 10_000, + request: request(), + session: session(), + }), + ).toEqual({ kind: "tier0-only" }); + }); + it("trains, saves, loads, and applies a learned router model", () => { const lowConfidenceInput = { classified: classified("age-question", 0.25), diff --git a/src/skills/onboarding/runtime/router.ts b/src/skills/onboarding/runtime/router.ts index 10e8955..41c4469 100644 --- a/src/skills/onboarding/runtime/router.ts +++ b/src/skills/onboarding/runtime/router.ts @@ -3,6 +3,10 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname } from "node:path"; import type { AgentRequest } from "@agent/types"; +import { + getAgentCostAwareTierEnabled, + getAlertThresholdP95LatencyMs, +} from "../../../config.js"; import type { ClassifiedIntent, DerivedState, @@ -58,8 +62,11 @@ export interface RouterTrainingSample { export interface RouterInput { classified: ClassifiedIntent; derived: DerivedState; + intentRiskTier?: "low" | "medium" | "high" | undefined; + recentLatencyP95Ms?: number | undefined; request: AgentRequest; session: OnboardingSession | null; + userEscalationCount?: number | undefined; } export const ROUTER_FEATURE_NAMES = [ @@ -294,6 +301,43 @@ function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean { ); } +function isLatencyDegraded(input: RouterInput): boolean { + const p95 = input.recentLatencyP95Ms; + return ( + typeof p95 === "number" && + Number.isFinite(p95) && + p95 > getAlertThresholdP95LatencyMs("onboarding") + ); +} + +function maybeDecideCostAwareRouting(input: RouterInput): RoutingDecision | null { + if (!getAgentCostAwareTierEnabled()) { + return null; + } + + const confidence = clamp01(input.classified.confidence); + const riskTier = input.intentRiskTier ?? "medium"; + const escalationCount = Math.max(0, input.userEscalationCount ?? 0); + const lowLatency = !isLatencyDegraded(input); + + if (confidence < 0.5 || (riskTier === "high" && escalationCount > 3)) { + if (input.derived.responseMode === "llm-with-tools") { + return { kind: "tier1-with-tools" }; + } + return { kind: "tier1-direct", skipPlanner: false }; + } + + if (!lowLatency) { + return { kind: "tier0-only" }; + } + + if (confidence >= 0.9 && riskTier === "low") { + return { kind: "tier0-only" }; + } + + return null; +} + export function decidePreGenerationRouting(input: RouterInput): RoutingDecision { const { classified, derived, request, session } = input; const { confidence, intent } = classified; @@ -334,6 +378,11 @@ export function decidePreGenerationRouting(input: RouterInput): RoutingDecision }; } + const costAwareDecision = maybeDecideCostAwareRouting(input); + if (costAwareDecision) { + return costAwareDecision; + } + if ( phase.kind === "nickname-rejected" || intent.kind === "discord-verify-question" || diff --git a/src/skills/onboarding/skill.ts b/src/skills/onboarding/skill.ts index d20c6bc..ca22e8a 100644 --- a/src/skills/onboarding/skill.ts +++ b/src/skills/onboarding/skill.ts @@ -1,6 +1,7 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types'; import type { GarnetClient } from '@common/garnet'; import type { NatsClient } from '@common/nats'; +import type { EscalationHistoryService } from '../../memory/escalation-history.js'; import type { MemoryService } from '../../memory/memory-types.js'; import { buildLongTermSummary, @@ -118,6 +119,7 @@ export class OnboardingSkill implements Skill { shadowEvalPublisher?: ShadowEvalPublisher, private readonly natsClient?: Pick, private readonly memoryService?: MemoryService, + private readonly escalationHistoryService?: EscalationHistoryService, ) { this.classifier = new OnboardingIntentClassifier(llmClient); this.blackboardStore = new OnboardingBlackboardStore( @@ -186,13 +188,17 @@ export class OnboardingSkill implements Skill { const deriveStep = trace?.startStep('derive'); const derived = deriveState(session, request.trigger, classified.intent); + const escalationHistory = await this.loadEscalationHistory(request.userId); const routerMode = getAgentRouterMode(); if (routerMode === 'static' || routerMode === 'learned-v1') { const routerInput = { classified, derived, + intentRiskTier: this.capabilityCard().riskTier, + recentLatencyP95Ms: this.getRecentLatencyP95Ms(request), request, session, + userEscalationCount: escalationHistory?.totalEscalations ?? 0, }; derived.routingDecision = routerMode === 'learned-v1' @@ -353,6 +359,26 @@ export class OnboardingSkill implements Skill { return normalizeOnboardingSession(rawSession); } + private async loadEscalationHistory(userId: string) { + if (!this.escalationHistoryService) { + return null; + } + try { + return await this.escalationHistoryService.load(userId); + } catch (error) { + getRequestLogger().warn( + { err: error }, + '[OnboardingSkill] Failed to load escalation history', + ); + return null; + } + } + + private getRecentLatencyP95Ms(request: AgentRequest): number | undefined { + const raw = request.context?.['recentLatencyP95Ms']; + return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined; + } + private async decideEarlyAbstention( request: AgentRequest, classified: ClassifiedIntent, diff --git a/src/skills/supervisor/routing-rules.ts b/src/skills/supervisor/routing-rules.ts index 71c899b..7313302 100644 --- a/src/skills/supervisor/routing-rules.ts +++ b/src/skills/supervisor/routing-rules.ts @@ -1,17 +1,53 @@ import type { HandoffPacket } from "@agent/handoff-packet"; -export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null { - const text = [ +export interface SupervisorRuleRoute { + confidence: number; + target: string | null; +} + +const RULES = [ + { + pattern: /(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu, + target: "moderation", + }, + { + pattern: /(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu, + target: "onboarding", + }, +] as const; + +function packetText(packet: HandoffPacket | undefined): string { + return [ packet?.userGoal, packet?.summary, packet?.handoffReason, ].join(" "); +} - if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) { - return "moderation"; +export function routeSupervisorByRules( + packet: HandoffPacket | undefined, +): SupervisorRuleRoute { + const text = packetText(packet); + const matches = RULES.filter((rule) => rule.pattern.test(text)); + + if (matches.length === 1) { + return { + confidence: 0.9, + target: matches[0]?.target ?? null, + }; } - if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) { - return "onboarding"; + if (matches.length > 1) { + return { + confidence: 0.6, + target: matches[0]?.target ?? null, + }; } - return null; + return { + confidence: 0, + target: null, + }; +} + +export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null { + return routeSupervisorByRules(packet).target; } diff --git a/src/skills/supervisor/skill.test.ts b/src/skills/supervisor/skill.test.ts index c18460d..735397f 100644 --- a/src/skills/supervisor/skill.test.ts +++ b/src/skills/supervisor/skill.test.ts @@ -119,7 +119,7 @@ describe("SupervisorSkill", () => { expect(response.text).toContain("운영자"); expect(response.metadata).toMatchObject({ - routedBy: "rules", + routedBy: "regex-fast-path", source: "template", supervisorReason: "max-hop", suggestedTarget: "onboarding", @@ -149,6 +149,7 @@ describe("SupervisorSkill", () => { }); it("uses LLM routing when the decision is confident and valid", async () => { + vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false"); const garnet = new FakeGarnet(); const nats = new FakeNats(); const skill = new SupervisorSkill( @@ -189,6 +190,45 @@ describe("SupervisorSkill", () => { }); }); + it("uses regex fast-path before calling the supervisor LLM", async () => { + const garnet = new FakeGarnet(); + const nats = new FakeNats(); + const llm = { + chatCompletion: vi.fn(), + }; + const skill = new SupervisorSkill( + garnet, + nats, + llm as unknown as Pick, + registry, + ); + const packet = buildHandoffPacket({ + handoff: { + confidence: 0.7, + evidenceRefs: [], + expectedOutput: "text", + reason: "out-of-scope", + risk: "medium", + targetAgent: "supervisor", + taskSpec: "광고 규칙 문의", + }, + request: baseRequest, + response: sourceResponse, + sourceChain: ["onboarding", "supervisor"], + }); + + const response = await skill.handle( + { ...baseRequest, context: { handoffPacket: packet } }, + [], + ); + + expect(llm.chatCompletion).not.toHaveBeenCalled(); + expect(response.metadata).toMatchObject({ + routedBy: "regex-fast-path", + suggestedTarget: "moderation", + }); + }); + it("uses safe LLM userMessage for supervisor replies", async () => { const garnet = new FakeGarnet(); const nats = new FakeNats(); diff --git a/src/skills/supervisor/skill.ts b/src/skills/supervisor/skill.ts index 55c57a2..f79781a 100644 --- a/src/skills/supervisor/skill.ts +++ b/src/skills/supervisor/skill.ts @@ -3,6 +3,7 @@ import type { AgentResponse, ConversationMessage, } from "@agent/types"; +import { trace } from "@opentelemetry/api"; import { readHandoffPacket } from "@agent/handoff-packet"; import type { HandoffPacket } from "@agent/handoff-packet"; import type { GarnetClient } from "@common/garnet"; @@ -19,6 +20,8 @@ import type { Skill } from "@skills/skill"; import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader"; import { getLogger } from "@obs/logger"; import { + getAgentSupervisorFastPathEnabled, + getSupervisorFastPathConfidence, getSupervisorLlmConfidenceThreshold, getSupervisorLlmEnabled, getSupervisorLlmResponseEnabled, @@ -28,7 +31,7 @@ import { buildSupervisorUserPrompt, } from "./capability-prompt.js"; import { appendSupervisorReview } from "./review-queue.js"; -import { routeSupervisorTarget } from "./routing-rules.js"; +import { routeSupervisorByRules } from "./routing-rules.js"; import { supervisorRoutingSchema, type SupervisorRoutingDecision, @@ -114,13 +117,30 @@ export class SupervisorSkill implements Skill { ): Promise { const packet = readHandoffPacket(request); const escalationHistory = await this.loadEscalationHistory(request.userId); - const llmDecision = await this.routeByLlm(packet, escalationHistory); + const rulesDecision = routeSupervisorByRules(packet); + const useRegexFastPath = + getAgentSupervisorFastPathEnabled() && + rulesDecision.target !== null && + rulesDecision.confidence >= getSupervisorFastPathConfidence() && + this.isKnownTarget(rulesDecision.target); + const llmDecision = useRegexFastPath + ? null + : await this.routeByLlm(packet, escalationHistory); const useLlmRoute = llmDecision !== null && llmDecision.confidence >= getSupervisorLlmConfidenceThreshold(); const target = useLlmRoute ? llmDecision.target - : routeSupervisorTarget(packet); + : rulesDecision.target; + const routedBy = useRegexFastPath + ? "regex-fast-path" + : useLlmRoute + ? "llm" + : "rules-fallback"; + trace.getActiveSpan()?.setAttributes({ + "supervisor.routed_by": routedBy, + "supervisor.rules_confidence": rulesDecision.confidence, + }); const reason = packet?.handoffReason ?? "out-of-scope"; const queuedAt = new Date().toISOString(); @@ -169,7 +189,8 @@ export class SupervisorSkill implements Skill { return { confidence: 0.6, metadata: { - routedBy: useLlmRoute ? "llm" : "rules", + routedBy, + rulesConfidence: rulesDecision.confidence, source: userMessage.source, supervisorReason: useLlmRoute ? llmDecision.reason : reason, supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0, @@ -234,7 +255,10 @@ export class SupervisorSkill implements Skill { private isKnownTarget(target: string): boolean { return ( - target === "human-review" || Boolean(this.skillRegistry?.get(target)) + target === "human-review" || + target === "onboarding" || + target === "moderation" || + Boolean(this.skillRegistry?.get(target)) ); }