Add cost-aware routing and LLM budget tracking
- BudgetTracker tracks LLM token cost separately from tool cost - LlmClient auto-records cost into request-scoped budget - Router selects tier from classify confidence, intent risk, escalation count - Supervisor short-circuits via regex fast-path when rule confidence is high - Onboarding skill loads user escalation history to inform routing
This commit is contained in:
1 parent
211de18476
commit
21df14cd00
16 files changed
+473
-22
No files matched your search
@@ -59,6 +59,30 @@ describe("BudgetTracker", () => {
|
||||
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
|
||||
});
|
||||
|
||||
it("tracks LLM and tool costs separately while enforcing the total budget", () => {
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: 0.0055,
|
||||
maxHandoffs: 5,
|
||||
maxSameAgentVisits: 5,
|
||||
maxToolCalls: 5,
|
||||
});
|
||||
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
budget.recordLlmCost(0.001);
|
||||
}
|
||||
budget.recordToolCall(0.0005);
|
||||
|
||||
expect(budget.snapshot()).toMatchObject({
|
||||
estimatedCostUsd: 0.0055,
|
||||
totalCostUsd: 0.0055,
|
||||
totalLlmCostUsd: 0.005,
|
||||
totalToolCostUsd: 0.0005,
|
||||
toolCalls: 1,
|
||||
});
|
||||
expect(budget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
|
||||
});
|
||||
|
||||
it("can be disabled", () => {
|
||||
const budget = new BudgetTracker({
|
||||
enabled: false,
|
||||
@@ -71,6 +95,7 @@ describe("BudgetTracker", () => {
|
||||
budget.recordVisit("supervisor");
|
||||
budget.recordHandoff("moderation");
|
||||
budget.recordToolCall(1);
|
||||
budget.recordLlmCost(1);
|
||||
|
||||
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
|
||||
});
|
||||
|
||||
+23
-4
@@ -20,13 +20,17 @@ export interface BudgetSnapshot {
|
||||
maxHandoffs: number;
|
||||
maxSameAgentVisits: number;
|
||||
maxToolCalls: number;
|
||||
totalCostUsd: number;
|
||||
totalLlmCostUsd: number;
|
||||
totalToolCostUsd: number;
|
||||
toolCalls: number;
|
||||
visits: Record<string, number>;
|
||||
}
|
||||
|
||||
export class BudgetTracker {
|
||||
private estimatedCostUsd = 0;
|
||||
private handoffs = 0;
|
||||
private totalLlmCostUsd = 0;
|
||||
private totalToolCostUsd = 0;
|
||||
private toolCalls = 0;
|
||||
private readonly visits = new Map<string, number>();
|
||||
|
||||
@@ -53,7 +57,7 @@ export class BudgetTracker {
|
||||
if (this.toolCalls >= this.limits.maxToolCalls) {
|
||||
return "tool-budget";
|
||||
}
|
||||
if (this.estimatedCostUsd >= this.limits.maxCostUsd) {
|
||||
if (this.totalCostUsd() >= this.limits.maxCostUsd) {
|
||||
return "cost-budget";
|
||||
}
|
||||
return "ok";
|
||||
@@ -66,7 +70,14 @@ export class BudgetTracker {
|
||||
|
||||
recordToolCall(estimatedCostUsd = 0): void {
|
||||
this.toolCalls += 1;
|
||||
this.estimatedCostUsd += estimatedCostUsd;
|
||||
this.totalToolCostUsd += estimatedCostUsd;
|
||||
}
|
||||
|
||||
recordLlmCost(costUsd: number): void {
|
||||
if (!Number.isFinite(costUsd) || costUsd <= 0) {
|
||||
return;
|
||||
}
|
||||
this.totalLlmCostUsd += costUsd;
|
||||
}
|
||||
|
||||
recordVisit(target: string): void {
|
||||
@@ -74,15 +85,23 @@ export class BudgetTracker {
|
||||
}
|
||||
|
||||
snapshot(): BudgetSnapshot {
|
||||
const totalCostUsd = this.totalCostUsd();
|
||||
return {
|
||||
estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)),
|
||||
estimatedCostUsd: Number(totalCostUsd.toFixed(6)),
|
||||
handoffs: this.handoffs,
|
||||
maxCostUsd: this.limits.maxCostUsd,
|
||||
maxHandoffs: this.limits.maxHandoffs,
|
||||
maxSameAgentVisits: this.limits.maxSameAgentVisits,
|
||||
maxToolCalls: this.limits.maxToolCalls,
|
||||
totalCostUsd: Number(totalCostUsd.toFixed(6)),
|
||||
totalLlmCostUsd: Number(this.totalLlmCostUsd.toFixed(6)),
|
||||
totalToolCostUsd: Number(this.totalToolCostUsd.toFixed(6)),
|
||||
toolCalls: this.toolCalls,
|
||||
visits: Object.fromEntries(this.visits.entries()),
|
||||
};
|
||||
}
|
||||
|
||||
private totalCostUsd(): number {
|
||||
return this.totalLlmCostUsd + this.totalToolCostUsd;
|
||||
}
|
||||
}
|
||||
@@ -90,7 +90,7 @@ export async function dispatch(
|
||||
return runWithContext(
|
||||
{
|
||||
...contextBase,
|
||||
budget: budget.snapshot(),
|
||||
budget,
|
||||
handoffChain: chain,
|
||||
handoffDepth: depth,
|
||||
},
|
||||
|
||||
@@ -5,10 +5,15 @@ import {
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getAgentBudgetLlmCostEnabled,
|
||||
getAgentCostAwareTierEnabled,
|
||||
getAgentSupervisorFastPathEnabled,
|
||||
getAgentToolSandboxEnabled,
|
||||
getEmbeddingModel,
|
||||
getEmbeddingProvider,
|
||||
getLongTermSummaryEnabled,
|
||||
getMemoryEvictionEnabled,
|
||||
getNatsAclMode,
|
||||
getPhoenixLogPrompts,
|
||||
getPhoenixOtlpEndpoint,
|
||||
getRagHybridEnabled,
|
||||
@@ -17,7 +22,9 @@ import {
|
||||
getRagEnabled,
|
||||
getSlackAlertWebhookUrl,
|
||||
getSlackWebhookTimeoutMs,
|
||||
getSupervisorFastPathConfidence,
|
||||
getSupervisorLlmResponseEnabled,
|
||||
getToolAuditMaxLen,
|
||||
} from "./config.js";
|
||||
|
||||
const ENV_KEYS = [
|
||||
@@ -35,6 +42,13 @@ const ENV_KEYS = [
|
||||
"AGENT_RAG_RERANKER_TOP_K",
|
||||
"AGENT_LONG_TERM_SUMMARY_ENABLED",
|
||||
"AGENT_MEMORY_EVICTION_ENABLED",
|
||||
"AGENT_BUDGET_LLM_COST_ENABLED",
|
||||
"AGENT_COST_AWARE_TIER_ENABLED",
|
||||
"AGENT_SUPERVISOR_FAST_PATH_ENABLED",
|
||||
"SUPERVISOR_FAST_PATH_CONFIDENCE",
|
||||
"AGENT_TOOL_SANDBOX_ENABLED",
|
||||
"AGENT_TOOL_AUDIT_MAX_LEN",
|
||||
"AGENT_NATS_ACL_MODE",
|
||||
"EMBEDDING_MODEL",
|
||||
"EMBEDDING_PROVIDER",
|
||||
"PHOENIX_LOG_PROMPTS",
|
||||
@@ -129,4 +143,30 @@ describe("config", () => {
|
||||
|
||||
expect(getSupervisorLlmResponseEnabled()).toBe(false);
|
||||
});
|
||||
|
||||
it("reads operational excellence guardrail configuration", () => {
|
||||
expect(getAgentBudgetLlmCostEnabled()).toBe(true);
|
||||
expect(getAgentCostAwareTierEnabled()).toBe(true);
|
||||
expect(getAgentSupervisorFastPathEnabled()).toBe(true);
|
||||
expect(getSupervisorFastPathConfidence()).toBe(0.85);
|
||||
expect(getAgentToolSandboxEnabled()).toBe(false);
|
||||
expect(getToolAuditMaxLen()).toBe(100_000);
|
||||
expect(getNatsAclMode()).toBe("log-only");
|
||||
|
||||
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
|
||||
vi.stubEnv("AGENT_COST_AWARE_TIER_ENABLED", "false");
|
||||
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
|
||||
vi.stubEnv("SUPERVISOR_FAST_PATH_CONFIDENCE", "0.9");
|
||||
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
|
||||
vi.stubEnv("AGENT_TOOL_AUDIT_MAX_LEN", "42");
|
||||
vi.stubEnv("AGENT_NATS_ACL_MODE", "enforce");
|
||||
|
||||
expect(getAgentBudgetLlmCostEnabled()).toBe(false);
|
||||
expect(getAgentCostAwareTierEnabled()).toBe(false);
|
||||
expect(getAgentSupervisorFastPathEnabled()).toBe(false);
|
||||
expect(getSupervisorFastPathConfidence()).toBe(0.9);
|
||||
expect(getAgentToolSandboxEnabled()).toBe(true);
|
||||
expect(getToolAuditMaxLen()).toBe(42);
|
||||
expect(getNatsAclMode()).toBe("enforce");
|
||||
});
|
||||
});
|
||||
@@ -231,6 +231,38 @@ export function getAgentBudgetMaxCostUsd(): number {
|
||||
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
|
||||
}
|
||||
|
||||
export function getAgentBudgetLlmCostEnabled(): boolean {
|
||||
return getBoolean("AGENT_BUDGET_LLM_COST_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getAgentCostAwareTierEnabled(): boolean {
|
||||
return getBoolean("AGENT_COST_AWARE_TIER_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getAgentSupervisorFastPathEnabled(): boolean {
|
||||
return getBoolean("AGENT_SUPERVISOR_FAST_PATH_ENABLED", true);
|
||||
}
|
||||
|
||||
export function getSupervisorFastPathConfidence(): number {
|
||||
return getRatioNumber("SUPERVISOR_FAST_PATH_CONFIDENCE", 0.85);
|
||||
}
|
||||
|
||||
export function getAgentToolSandboxEnabled(): boolean {
|
||||
return getBoolean("AGENT_TOOL_SANDBOX_ENABLED", false);
|
||||
}
|
||||
|
||||
export function getToolAuditMaxLen(): number {
|
||||
return getPositiveNumber("AGENT_TOOL_AUDIT_MAX_LEN", 100_000);
|
||||
}
|
||||
|
||||
export function getNatsAclMode(): "off" | "log-only" | "enforce" {
|
||||
const raw = process.env["AGENT_NATS_ACL_MODE"]?.trim().toLowerCase();
|
||||
if (raw === "off" || raw === "log-only" || raw === "enforce") {
|
||||
return raw;
|
||||
}
|
||||
return "log-only";
|
||||
}
|
||||
|
||||
export function getAgentRuntimeVerifierEnabled(): boolean {
|
||||
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
|
||||
}
|
||||
|
||||
@@ -25,6 +25,8 @@ vi.mock("@obs/spans", () => ({
|
||||
}));
|
||||
|
||||
import { LlmClient } from "./llm-client.js";
|
||||
import { BudgetTracker } from "@agent/budget";
|
||||
import { runWithContext } from "@obs/context";
|
||||
import type { ChatCompletionResponse } from "./types.js";
|
||||
|
||||
function okResponse(
|
||||
@@ -168,4 +170,103 @@ describe("LlmClient endpoint normalization", () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("records LLM cost into the request budget", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(() =>
|
||||
Promise.resolve(
|
||||
okResponse("reply", {
|
||||
usage: {
|
||||
completion_tokens: 50,
|
||||
prompt_tokens: 100,
|
||||
total_tokens: 150,
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: 1,
|
||||
maxHandoffs: 5,
|
||||
maxSameAgentVisits: 5,
|
||||
maxToolCalls: 5,
|
||||
});
|
||||
const client = new LlmClient({
|
||||
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
|
||||
providerApiKeys: { openrouter: "test-key" },
|
||||
});
|
||||
|
||||
await runWithContext(
|
||||
{
|
||||
budget,
|
||||
chatroomId: "room-1",
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
startedAt: "2026-05-18T00:00:00.000Z",
|
||||
trigger: "user-message",
|
||||
userId: "user-1",
|
||||
},
|
||||
() =>
|
||||
client.chatCompletion([{ content: "hi", role: "user" }], {
|
||||
modelOverride: "gpt-5.3-codex-spark",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(budget.snapshot().totalLlmCostUsd).toBe(0.000045);
|
||||
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
"budget.llm_cost_usd": 0.000045,
|
||||
"budget.total_cost_usd": 0.000045,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not record LLM cost when budget cost tracking is disabled", async () => {
|
||||
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(() =>
|
||||
Promise.resolve(
|
||||
okResponse("reply", {
|
||||
usage: {
|
||||
completion_tokens: 50,
|
||||
prompt_tokens: 100,
|
||||
total_tokens: 150,
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
const budget = new BudgetTracker({
|
||||
enabled: true,
|
||||
maxCostUsd: 1,
|
||||
maxHandoffs: 5,
|
||||
maxSameAgentVisits: 5,
|
||||
maxToolCalls: 5,
|
||||
});
|
||||
const client = new LlmClient({
|
||||
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
|
||||
providerApiKeys: { openrouter: "test-key" },
|
||||
});
|
||||
|
||||
await runWithContext(
|
||||
{
|
||||
budget,
|
||||
chatroomId: "room-1",
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
startedAt: "2026-05-18T00:00:00.000Z",
|
||||
trigger: "user-message",
|
||||
userId: "user-1",
|
||||
},
|
||||
() =>
|
||||
client.chatCompletion([{ content: "hi", role: "user" }], {
|
||||
modelOverride: "gpt-5.3-codex-spark",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(budget.snapshot().totalLlmCostUsd).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
import { getContext, getRequestLogger } from "@obs/context";
|
||||
import { maskTraceContent, withLlmSpan } from "@obs/spans";
|
||||
import {
|
||||
getAgentBudgetLlmCostEnabled,
|
||||
getLlmHealthTimeoutMs,
|
||||
getLlmRequestTimeoutMs,
|
||||
getLlmTemperature,
|
||||
@@ -257,11 +258,22 @@ export class LlmClient {
|
||||
model: request.model,
|
||||
skill: getContext()?.skill,
|
||||
});
|
||||
if (getAgentBudgetLlmCostEnabled()) {
|
||||
getContext()?.budget?.recordLlmCost(costUsd);
|
||||
}
|
||||
const budgetSnapshot = getContext()?.budget?.snapshot();
|
||||
|
||||
span.setAttributes({
|
||||
...(shouldLogPrompts
|
||||
? buildCompletionTraceAttributes(result)
|
||||
: {}),
|
||||
...(budgetSnapshot
|
||||
? {
|
||||
"budget.llm_cost_usd": budgetSnapshot.totalLlmCostUsd,
|
||||
"budget.tool_cost_usd": budgetSnapshot.totalToolCostUsd,
|
||||
"budget.total_cost_usd": budgetSnapshot.totalCostUsd,
|
||||
}
|
||||
: {}),
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
?.map((choice) => choice.finish_reason ?? "unknown")
|
||||
.filter((reason): reason is string => Boolean(reason)),
|
||||
|
||||
@@ -3,12 +3,12 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import type { Logger } from 'pino';
|
||||
|
||||
import { getLogger } from './logger.js';
|
||||
import type { BudgetSnapshot } from '@agent/budget';
|
||||
import type { BudgetTracker } from '@agent/budget';
|
||||
|
||||
export interface RequestContext {
|
||||
chatroomId: string;
|
||||
deadlineAt?: string;
|
||||
budget?: BudgetSnapshot | undefined;
|
||||
budget?: BudgetTracker | undefined;
|
||||
handoffChain?: string[] | undefined;
|
||||
handoffDepth?: number | undefined;
|
||||
logger: Logger;
|
||||
|
||||
@@ -46,6 +46,9 @@ describe("escalation trigger", () => {
|
||||
maxHandoffs: 4,
|
||||
maxSameAgentVisits: 2,
|
||||
maxToolCalls: 20,
|
||||
totalCostUsd: 0,
|
||||
totalLlmCostUsd: 0,
|
||||
totalToolCostUsd: 0,
|
||||
toolCalls: 0,
|
||||
visits: {},
|
||||
},
|
||||
|
||||
@@ -16,5 +16,6 @@ export function createOnboardingSkill(deps: SkillDeps): Skill {
|
||||
deps.shadowEvalBus,
|
||||
deps.natsClient,
|
||||
deps.memoryService,
|
||||
deps.escalationHistoryService,
|
||||
);
|
||||
}
|
||||
@@ -102,9 +102,9 @@ describe("decidePreGenerationRouting", () => {
|
||||
},
|
||||
{
|
||||
confidence: 0.3,
|
||||
expected: { kind: "tier0-only" },
|
||||
expected: { kind: "tier1-direct", skipPlanner: false },
|
||||
intentKind: "off-topic-to-other" as const,
|
||||
name: "keeps low-confidence off-topic on tier0",
|
||||
name: "promotes low-confidence off-topic to tier1",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
@@ -319,6 +319,49 @@ describe("decidePreGenerationRouting", () => {
|
||||
).toEqual(expected);
|
||||
});
|
||||
|
||||
it("keeps high-confidence low-risk requests on tier0", () => {
|
||||
expect(
|
||||
decidePreGenerationRouting({
|
||||
classified: classified("age-question", 0.95),
|
||||
derived: derived({ kind: "deadline-near", remaining: ["photo"] }, intent("age-question"), {
|
||||
tier: 1,
|
||||
}),
|
||||
intentRiskTier: "low",
|
||||
request: request(),
|
||||
session: session(),
|
||||
userEscalationCount: 0,
|
||||
}),
|
||||
).toEqual({ kind: "tier0-only" });
|
||||
});
|
||||
|
||||
it("promotes low-confidence high-risk requests with escalation history", () => {
|
||||
expect(
|
||||
decidePreGenerationRouting({
|
||||
classified: classified("unknown", 0.49),
|
||||
derived: derived({ kind: "fresh" }, intent("unknown")),
|
||||
intentRiskTier: "high",
|
||||
request: request(),
|
||||
session: session(),
|
||||
userEscalationCount: 4,
|
||||
}),
|
||||
).toEqual({ kind: "tier1-direct", skipPlanner: false });
|
||||
});
|
||||
|
||||
it("prefers tier0 during latency degradation unless safety already promoted", () => {
|
||||
expect(
|
||||
decidePreGenerationRouting({
|
||||
classified: classified("age-question", 0.8),
|
||||
derived: derived({ kind: "deadline-near", remaining: ["nickname"] }, intent("age-question"), {
|
||||
tier: 1,
|
||||
}),
|
||||
intentRiskTier: "medium",
|
||||
recentLatencyP95Ms: 10_000,
|
||||
request: request(),
|
||||
session: session(),
|
||||
}),
|
||||
).toEqual({ kind: "tier0-only" });
|
||||
});
|
||||
|
||||
it("trains, saves, loads, and applies a learned router model", () => {
|
||||
const lowConfidenceInput = {
|
||||
classified: classified("age-question", 0.25),
|
||||
|
||||
@@ -3,6 +3,10 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import {
|
||||
getAgentCostAwareTierEnabled,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
} from "../../../config.js";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
@@ -58,8 +62,11 @@ export interface RouterTrainingSample {
|
||||
export interface RouterInput {
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState;
|
||||
intentRiskTier?: "low" | "medium" | "high" | undefined;
|
||||
recentLatencyP95Ms?: number | undefined;
|
||||
request: AgentRequest;
|
||||
session: OnboardingSession | null;
|
||||
userEscalationCount?: number | undefined;
|
||||
}
|
||||
|
||||
export const ROUTER_FEATURE_NAMES = [
|
||||
@@ -294,6 +301,43 @@ function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function isLatencyDegraded(input: RouterInput): boolean {
|
||||
const p95 = input.recentLatencyP95Ms;
|
||||
return (
|
||||
typeof p95 === "number" &&
|
||||
Number.isFinite(p95) &&
|
||||
p95 > getAlertThresholdP95LatencyMs("onboarding")
|
||||
);
|
||||
}
|
||||
|
||||
function maybeDecideCostAwareRouting(input: RouterInput): RoutingDecision | null {
|
||||
if (!getAgentCostAwareTierEnabled()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const confidence = clamp01(input.classified.confidence);
|
||||
const riskTier = input.intentRiskTier ?? "medium";
|
||||
const escalationCount = Math.max(0, input.userEscalationCount ?? 0);
|
||||
const lowLatency = !isLatencyDegraded(input);
|
||||
|
||||
if (confidence < 0.5 || (riskTier === "high" && escalationCount > 3)) {
|
||||
if (input.derived.responseMode === "llm-with-tools") {
|
||||
return { kind: "tier1-with-tools" };
|
||||
}
|
||||
return { kind: "tier1-direct", skipPlanner: false };
|
||||
}
|
||||
|
||||
if (!lowLatency) {
|
||||
return { kind: "tier0-only" };
|
||||
}
|
||||
|
||||
if (confidence >= 0.9 && riskTier === "low") {
|
||||
return { kind: "tier0-only" };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
|
||||
const { classified, derived, request, session } = input;
|
||||
const { confidence, intent } = classified;
|
||||
@@ -334,6 +378,11 @@ export function decidePreGenerationRouting(input: RouterInput): RoutingDecision
|
||||
};
|
||||
}
|
||||
|
||||
const costAwareDecision = maybeDecideCostAwareRouting(input);
|
||||
if (costAwareDecision) {
|
||||
return costAwareDecision;
|
||||
}
|
||||
|
||||
if (
|
||||
phase.kind === "nickname-rejected" ||
|
||||
intent.kind === "discord-verify-question" ||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
|
||||
import type { GarnetClient } from '@common/garnet';
|
||||
import type { NatsClient } from '@common/nats';
|
||||
import type { EscalationHistoryService } from '../../memory/escalation-history.js';
|
||||
import type { MemoryService } from '../../memory/memory-types.js';
|
||||
import {
|
||||
buildLongTermSummary,
|
||||
@@ -118,6 +119,7 @@ export class OnboardingSkill implements Skill {
|
||||
shadowEvalPublisher?: ShadowEvalPublisher,
|
||||
private readonly natsClient?: Pick<NatsClient, 'request'>,
|
||||
private readonly memoryService?: MemoryService,
|
||||
private readonly escalationHistoryService?: EscalationHistoryService,
|
||||
) {
|
||||
this.classifier = new OnboardingIntentClassifier(llmClient);
|
||||
this.blackboardStore = new OnboardingBlackboardStore(
|
||||
@@ -186,13 +188,17 @@ export class OnboardingSkill implements Skill {
|
||||
|
||||
const deriveStep = trace?.startStep('derive');
|
||||
const derived = deriveState(session, request.trigger, classified.intent);
|
||||
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
||||
const routerMode = getAgentRouterMode();
|
||||
if (routerMode === 'static' || routerMode === 'learned-v1') {
|
||||
const routerInput = {
|
||||
classified,
|
||||
derived,
|
||||
intentRiskTier: this.capabilityCard().riskTier,
|
||||
recentLatencyP95Ms: this.getRecentLatencyP95Ms(request),
|
||||
request,
|
||||
session,
|
||||
userEscalationCount: escalationHistory?.totalEscalations ?? 0,
|
||||
};
|
||||
derived.routingDecision =
|
||||
routerMode === 'learned-v1'
|
||||
@@ -353,6 +359,26 @@ export class OnboardingSkill implements Skill {
|
||||
return normalizeOnboardingSession(rawSession);
|
||||
}
|
||||
|
||||
private async loadEscalationHistory(userId: string) {
|
||||
if (!this.escalationHistoryService) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return await this.escalationHistoryService.load(userId);
|
||||
} catch (error) {
|
||||
getRequestLogger().warn(
|
||||
{ err: error },
|
||||
'[OnboardingSkill] Failed to load escalation history',
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private getRecentLatencyP95Ms(request: AgentRequest): number | undefined {
|
||||
const raw = request.context?.['recentLatencyP95Ms'];
|
||||
return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined;
|
||||
}
|
||||
|
||||
private async decideEarlyAbstention(
|
||||
request: AgentRequest,
|
||||
classified: ClassifiedIntent,
|
||||
|
||||
@@ -1,17 +1,53 @@
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
|
||||
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
|
||||
const text = [
|
||||
export interface SupervisorRuleRoute {
|
||||
confidence: number;
|
||||
target: string | null;
|
||||
}
|
||||
|
||||
const RULES = [
|
||||
{
|
||||
pattern: /(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu,
|
||||
target: "moderation",
|
||||
},
|
||||
{
|
||||
pattern: /(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu,
|
||||
target: "onboarding",
|
||||
},
|
||||
] as const;
|
||||
|
||||
function packetText(packet: HandoffPacket | undefined): string {
|
||||
return [
|
||||
packet?.userGoal,
|
||||
packet?.summary,
|
||||
packet?.handoffReason,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) {
|
||||
return "moderation";
|
||||
export function routeSupervisorByRules(
|
||||
packet: HandoffPacket | undefined,
|
||||
): SupervisorRuleRoute {
|
||||
const text = packetText(packet);
|
||||
const matches = RULES.filter((rule) => rule.pattern.test(text));
|
||||
|
||||
if (matches.length === 1) {
|
||||
return {
|
||||
confidence: 0.9,
|
||||
target: matches[0]?.target ?? null,
|
||||
};
|
||||
}
|
||||
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) {
|
||||
return "onboarding";
|
||||
if (matches.length > 1) {
|
||||
return {
|
||||
confidence: 0.6,
|
||||
target: matches[0]?.target ?? null,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
return {
|
||||
confidence: 0,
|
||||
target: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
|
||||
return routeSupervisorByRules(packet).target;
|
||||
}
|
||||
@@ -119,7 +119,7 @@ describe("SupervisorSkill", () => {
|
||||
|
||||
expect(response.text).toContain("운영자");
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "rules",
|
||||
routedBy: "regex-fast-path",
|
||||
source: "template",
|
||||
supervisorReason: "max-hop",
|
||||
suggestedTarget: "onboarding",
|
||||
@@ -149,6 +149,7 @@ describe("SupervisorSkill", () => {
|
||||
});
|
||||
|
||||
it("uses LLM routing when the decision is confident and valid", async () => {
|
||||
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const skill = new SupervisorSkill(
|
||||
@@ -189,6 +190,45 @@ describe("SupervisorSkill", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("uses regex fast-path before calling the supervisor LLM", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
const llm = {
|
||||
chatCompletion: vi.fn(),
|
||||
};
|
||||
const skill = new SupervisorSkill(
|
||||
garnet,
|
||||
nats,
|
||||
llm as unknown as Pick<LlmClient, "chatCompletion">,
|
||||
registry,
|
||||
);
|
||||
const packet = buildHandoffPacket({
|
||||
handoff: {
|
||||
confidence: 0.7,
|
||||
evidenceRefs: [],
|
||||
expectedOutput: "text",
|
||||
reason: "out-of-scope",
|
||||
risk: "medium",
|
||||
targetAgent: "supervisor",
|
||||
taskSpec: "광고 규칙 문의",
|
||||
},
|
||||
request: baseRequest,
|
||||
response: sourceResponse,
|
||||
sourceChain: ["onboarding", "supervisor"],
|
||||
});
|
||||
|
||||
const response = await skill.handle(
|
||||
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||
[],
|
||||
);
|
||||
|
||||
expect(llm.chatCompletion).not.toHaveBeenCalled();
|
||||
expect(response.metadata).toMatchObject({
|
||||
routedBy: "regex-fast-path",
|
||||
suggestedTarget: "moderation",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses safe LLM userMessage for supervisor replies", async () => {
|
||||
const garnet = new FakeGarnet();
|
||||
const nats = new FakeNats();
|
||||
|
||||
@@ -3,6 +3,7 @@ import type {
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
@@ -19,6 +20,8 @@ import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import {
|
||||
getAgentSupervisorFastPathEnabled,
|
||||
getSupervisorFastPathConfidence,
|
||||
getSupervisorLlmConfidenceThreshold,
|
||||
getSupervisorLlmEnabled,
|
||||
getSupervisorLlmResponseEnabled,
|
||||
@@ -28,7 +31,7 @@ import {
|
||||
buildSupervisorUserPrompt,
|
||||
} from "./capability-prompt.js";
|
||||
import { appendSupervisorReview } from "./review-queue.js";
|
||||
import { routeSupervisorTarget } from "./routing-rules.js";
|
||||
import { routeSupervisorByRules } from "./routing-rules.js";
|
||||
import {
|
||||
supervisorRoutingSchema,
|
||||
type SupervisorRoutingDecision,
|
||||
@@ -114,13 +117,30 @@ export class SupervisorSkill implements Skill {
|
||||
): Promise<AgentResponse> {
|
||||
const packet = readHandoffPacket(request);
|
||||
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
||||
const llmDecision = await this.routeByLlm(packet, escalationHistory);
|
||||
const rulesDecision = routeSupervisorByRules(packet);
|
||||
const useRegexFastPath =
|
||||
getAgentSupervisorFastPathEnabled() &&
|
||||
rulesDecision.target !== null &&
|
||||
rulesDecision.confidence >= getSupervisorFastPathConfidence() &&
|
||||
this.isKnownTarget(rulesDecision.target);
|
||||
const llmDecision = useRegexFastPath
|
||||
? null
|
||||
: await this.routeByLlm(packet, escalationHistory);
|
||||
const useLlmRoute =
|
||||
llmDecision !== null &&
|
||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||
const target = useLlmRoute
|
||||
? llmDecision.target
|
||||
: routeSupervisorTarget(packet);
|
||||
: rulesDecision.target;
|
||||
const routedBy = useRegexFastPath
|
||||
? "regex-fast-path"
|
||||
: useLlmRoute
|
||||
? "llm"
|
||||
: "rules-fallback";
|
||||
trace.getActiveSpan()?.setAttributes({
|
||||
"supervisor.routed_by": routedBy,
|
||||
"supervisor.rules_confidence": rulesDecision.confidence,
|
||||
});
|
||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||
|
||||
const queuedAt = new Date().toISOString();
|
||||
@@ -169,7 +189,8 @@ export class SupervisorSkill implements Skill {
|
||||
return {
|
||||
confidence: 0.6,
|
||||
metadata: {
|
||||
routedBy: useLlmRoute ? "llm" : "rules",
|
||||
routedBy,
|
||||
rulesConfidence: rulesDecision.confidence,
|
||||
source: userMessage.source,
|
||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
|
||||
@@ -234,7 +255,10 @@ export class SupervisorSkill implements Skill {
|
||||
|
||||
private isKnownTarget(target: string): boolean {
|
||||
return (
|
||||
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
||||
target === "human-review" ||
|
||||
target === "onboarding" ||
|
||||
target === "moderation" ||
|
||||
Boolean(this.skillRegistry?.get(target))
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user