Compare commits

...
3 Commits
Author SHA1 Message Date
dahyeon bc0f8e0773 Add secret coverage and NATS subject ACL
- KNOWN_SECRET_KEYS now covers LLM, embedding, and Phoenix API keys
- Logger redaction paths extended to cover the new credential keys
- nats-skill-acl maps each skill to allowed publish/subscribe subjects
- NATS publish runs ACL check with log-only/enforce/off modes
2026-05-19 01:28:54 -07:00
dahyeon d011a2829c Add tool permission sandbox and audit stream
- Tool interface declares required permissions via requires field
- scoped-context builds a least-privileged ToolContext per call
- ToolAuditLogger appends each tool invocation to a Garnet stream
- Operations CLI gains tool-audit:list and tool-audit:show commands
- All onboarding and moderation tools annotated with explicit requires
2026-05-19 01:28:54 -07:00
dahyeon 21df14cd00 Add cost-aware routing and LLM budget tracking
- BudgetTracker tracks LLM token cost separately from tool cost
- LlmClient auto-records cost into request-scoped budget
- Router selects tier from classify confidence, intent risk, escalation count
- Supervisor short-circuits via regex fast-path when rule confidence is high
- Onboarding skill loads user escalation history to inform routing
2026-05-19 01:28:54 -07:00
36 changed files with 1249 additions and 25 deletions

No files matched your search

+2
View File
@@ -36,6 +36,8 @@
"review:show": "node --import tsx src/operations/cli.ts review:show", "review:show": "node --import tsx src/operations/cli.ts review:show",
"review:resolve": "node --import tsx src/operations/cli.ts review:resolve", "review:resolve": "node --import tsx src/operations/cli.ts review:resolve",
"review:stats": "node --import tsx src/operations/cli.ts review:stats", "review:stats": "node --import tsx src/operations/cli.ts review:stats",
"tool-audit:list": "node --import tsx src/operations/cli.ts tool-audit:list",
"tool-audit:show": "node --import tsx src/operations/cli.ts tool-audit:show",
"start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"", "start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"",
"start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"", "start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"",
"start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"", "start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"",
+25
View File
@@ -59,6 +59,30 @@ describe("BudgetTracker", () => {
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget"); expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
}); });
it("tracks LLM and tool costs separately while enforcing the total budget", () => {
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 0.0055,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 5,
});
for (let index = 0; index < 5; index += 1) {
budget.recordLlmCost(0.001);
}
budget.recordToolCall(0.0005);
expect(budget.snapshot()).toMatchObject({
estimatedCostUsd: 0.0055,
totalCostUsd: 0.0055,
totalLlmCostUsd: 0.005,
totalToolCostUsd: 0.0005,
toolCalls: 1,
});
expect(budget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
});
it("can be disabled", () => { it("can be disabled", () => {
const budget = new BudgetTracker({ const budget = new BudgetTracker({
enabled: false, enabled: false,
@@ -71,6 +95,7 @@ describe("BudgetTracker", () => {
budget.recordVisit("supervisor"); budget.recordVisit("supervisor");
budget.recordHandoff("moderation"); budget.recordHandoff("moderation");
budget.recordToolCall(1); budget.recordToolCall(1);
budget.recordLlmCost(1);
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok"); expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
}); });
+23 -4
View File
@@ -20,13 +20,17 @@ export interface BudgetSnapshot {
maxHandoffs: number; maxHandoffs: number;
maxSameAgentVisits: number; maxSameAgentVisits: number;
maxToolCalls: number; maxToolCalls: number;
totalCostUsd: number;
totalLlmCostUsd: number;
totalToolCostUsd: number;
toolCalls: number; toolCalls: number;
visits: Record<string, number>; visits: Record<string, number>;
} }
export class BudgetTracker { export class BudgetTracker {
private estimatedCostUsd = 0;
private handoffs = 0; private handoffs = 0;
private totalLlmCostUsd = 0;
private totalToolCostUsd = 0;
private toolCalls = 0; private toolCalls = 0;
private readonly visits = new Map<string, number>(); private readonly visits = new Map<string, number>();
@@ -53,7 +57,7 @@ export class BudgetTracker {
if (this.toolCalls >= this.limits.maxToolCalls) { if (this.toolCalls >= this.limits.maxToolCalls) {
return "tool-budget"; return "tool-budget";
} }
if (this.estimatedCostUsd >= this.limits.maxCostUsd) { if (this.totalCostUsd() >= this.limits.maxCostUsd) {
return "cost-budget"; return "cost-budget";
} }
return "ok"; return "ok";
@@ -66,7 +70,14 @@ export class BudgetTracker {
recordToolCall(estimatedCostUsd = 0): void { recordToolCall(estimatedCostUsd = 0): void {
this.toolCalls += 1; this.toolCalls += 1;
this.estimatedCostUsd += estimatedCostUsd; this.totalToolCostUsd += estimatedCostUsd;
}
recordLlmCost(costUsd: number): void {
if (!Number.isFinite(costUsd) || costUsd <= 0) {
return;
}
this.totalLlmCostUsd += costUsd;
} }
recordVisit(target: string): void { recordVisit(target: string): void {
@@ -74,15 +85,23 @@ export class BudgetTracker {
} }
snapshot(): BudgetSnapshot { snapshot(): BudgetSnapshot {
const totalCostUsd = this.totalCostUsd();
return { return {
estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)), estimatedCostUsd: Number(totalCostUsd.toFixed(6)),
handoffs: this.handoffs, handoffs: this.handoffs,
maxCostUsd: this.limits.maxCostUsd, maxCostUsd: this.limits.maxCostUsd,
maxHandoffs: this.limits.maxHandoffs, maxHandoffs: this.limits.maxHandoffs,
maxSameAgentVisits: this.limits.maxSameAgentVisits, maxSameAgentVisits: this.limits.maxSameAgentVisits,
maxToolCalls: this.limits.maxToolCalls, maxToolCalls: this.limits.maxToolCalls,
totalCostUsd: Number(totalCostUsd.toFixed(6)),
totalLlmCostUsd: Number(this.totalLlmCostUsd.toFixed(6)),
totalToolCostUsd: Number(this.totalToolCostUsd.toFixed(6)),
toolCalls: this.toolCalls, toolCalls: this.toolCalls,
visits: Object.fromEntries(this.visits.entries()), visits: Object.fromEntries(this.visits.entries()),
}; };
} }
private totalCostUsd(): number {
return this.totalLlmCostUsd + this.totalToolCostUsd;
}
} }
+1 -1
View File
@@ -90,7 +90,7 @@ export async function dispatch(
return runWithContext( return runWithContext(
{ {
...contextBase, ...contextBase,
budget: budget.snapshot(), budget,
handoffChain: chain, handoffChain: chain,
handoffDepth: depth, handoffDepth: depth,
}, },
+5
View File
@@ -9,6 +9,11 @@ export const KNOWN_SECRET_KEYS = [
"GARNET_PASSWORD", "GARNET_PASSWORD",
"MONGODB_CONNECTION_STRING", "MONGODB_CONNECTION_STRING",
"MONGODB_PROJECT_ID", "MONGODB_PROJECT_ID",
"OPENROUTER_API_KEY",
"AGENT_LLM_API_KEY",
"OPENAI_API_KEY",
"EMBEDDING_API_KEY",
"PHOENIX_API_KEY",
"SLACK_ALERT_WEBHOOK_URL", "SLACK_ALERT_WEBHOOK_URL",
] as const; ] as const;
+41
View File
@@ -0,0 +1,41 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { runWithContext } from "../observability/context.js";
import { NatsClient } from "./nats.js";
import { AGENT_ALERT_SUBJECT, DISCORD_MEMBER_LOOKUP_SUBJECT } from "./nats-subjects.js";
import { checkPublishAllowed } from "./nats-skill-acl.js";
describe("NATS skill ACL", () => {
afterEach(() => {
vi.unstubAllEnvs();
});
it("allows declared publish subjects", () => {
expect(checkPublishAllowed("onboarding", DISCORD_MEMBER_LOOKUP_SUBJECT)).toBe(
true,
);
expect(checkPublishAllowed("supervisor", AGENT_ALERT_SUBJECT)).toBe(true);
expect(checkPublishAllowed("onboarding", AGENT_ALERT_SUBJECT)).toBe(false);
});
it("enforces denied publish subjects when enabled", async () => {
vi.stubEnv("AGENT_NATS_ACL_MODE", "enforce");
const client = new NatsClient();
await expect(
runWithContext(
{
chatroomId: "room-1",
requestId: "req-1",
skill: "onboarding",
startedAt: "2026-05-18T00:00:00.000Z",
trigger: "user-message",
userId: "user-1",
},
() => client.publish(AGENT_ALERT_SUBJECT, { ok: true }),
),
).rejects.toThrow(
`NATS publish denied for skill onboarding on subject ${AGENT_ALERT_SUBJECT}`,
);
});
});
+54
View File
@@ -0,0 +1,54 @@
import {
AGENT_ALERT_SUBJECT,
DISCORD_MEMBER_LOOKUP_SUBJECT,
} from "./nats-subjects.js";
export interface SkillNatsAcl {
skill: string;
allowedPublishSubjects: string[];
allowedSubscribeSubjects: string[];
}
export const DEFAULT_NATS_ACL: SkillNatsAcl[] = [
{
skill: "onboarding",
allowedPublishSubjects: [DISCORD_MEMBER_LOOKUP_SUBJECT],
allowedSubscribeSubjects: [],
},
{
skill: "moderation",
allowedPublishSubjects: [],
allowedSubscribeSubjects: [],
},
{
skill: "supervisor",
allowedPublishSubjects: [AGENT_ALERT_SUBJECT],
allowedSubscribeSubjects: [],
},
];
function wildcardToRegExp(pattern: string): RegExp {
const escaped = pattern
.split("*")
.map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"))
.join(".*");
return new RegExp(`^${escaped}$`, "u");
}
function matchesAny(subject: string, patterns: readonly string[]): boolean {
return patterns.some((pattern) => wildcardToRegExp(pattern).test(subject));
}
function getAcl(skill: string): SkillNatsAcl | undefined {
return DEFAULT_NATS_ACL.find((entry) => entry.skill === skill);
}
export function checkPublishAllowed(skill: string, subject: string): boolean {
const acl = getAcl(skill);
return Boolean(acl && matchesAny(subject, acl.allowedPublishSubjects));
}
export function checkSubscribeAllowed(skill: string, subject: string): boolean {
const acl = getAcl(skill);
return Boolean(acl && matchesAny(subject, acl.allowedSubscribeSubjects));
}
+39
View File
@@ -6,7 +6,13 @@ import {
type Subscription, type Subscription,
} from "nats"; } from "nats";
import { getSecret } from "@common/secret-manager"; import { getSecret } from "@common/secret-manager";
import { getNatsAclMode } from "../config.js";
import { getContext } from "../observability/context.js";
import { getLogger } from "@obs/logger"; import { getLogger } from "@obs/logger";
import {
checkPublishAllowed,
checkSubscribeAllowed,
} from "./nats-skill-acl.js";
export interface NatsMessageContext<T = unknown> { export interface NatsMessageContext<T = unknown> {
data: T; data: T;
@@ -81,6 +87,7 @@ export class NatsClient {
subject: string, subject: string,
handler: (msg: Msg) => void | Promise<void>, handler: (msg: Msg) => void | Promise<void>,
): Promise<Subscription> { ): Promise<Subscription> {
this.checkSubjectAcl("subscribe", subject);
const subscription = this.getConnection().subscribe(subject); const subscription = this.getConnection().subscribe(subject);
void (async () => { void (async () => {
@@ -101,6 +108,7 @@ export class NatsClient {
} }
async publish<T = unknown>(subject: string, data: T): Promise<void> { async publish<T = unknown>(subject: string, data: T): Promise<void> {
this.checkSubjectAcl("publish", subject);
const client = this.getConnection(); const client = this.getConnection();
client.publish(subject, this.stringCodec.encode(JSON.stringify(data))); client.publish(subject, this.stringCodec.encode(JSON.stringify(data)));
await client.flush(); await client.flush();
@@ -111,6 +119,7 @@ export class NatsClient {
data: TRequest, data: TRequest,
options?: { timeoutMs?: number | undefined }, options?: { timeoutMs?: number | undefined },
): Promise<TResponse> { ): Promise<TResponse> {
this.checkSubjectAcl("publish", subject);
const requestOptions = const requestOptions =
options?.timeoutMs !== undefined options?.timeoutMs !== undefined
? { ? {
@@ -126,6 +135,36 @@ export class NatsClient {
return this.decodeMessage<TResponse>(response); return this.decodeMessage<TResponse>(response);
} }
private checkSubjectAcl(
operation: "publish" | "subscribe",
subject: string,
): void {
const mode = getNatsAclMode();
if (mode === "off") {
return;
}
const skill = getContext()?.skill;
if (!skill) {
return;
}
const allowed =
operation === "publish"
? checkPublishAllowed(skill, subject)
: checkSubscribeAllowed(skill, subject);
if (allowed) {
return;
}
const payload = { mode, operation, skill, subject };
if (mode === "enforce") {
this.logger.error(payload, "[NATS ACL] Subject denied");
throw new Error(
`NATS ${operation} denied for skill ${skill} on subject ${subject}`,
);
}
this.logger.warn(payload, "[NATS ACL] Subject would be denied");
}
respond(msg: Msg, data: unknown): Promise<void> { respond(msg: Msg, data: unknown): Promise<void> {
if (!msg.reply) { if (!msg.reply) {
this.logger.warn( this.logger.warn(
+40
View File
@@ -5,10 +5,15 @@ import {
getAlertThresholdEvalScoreDropPct, getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate, getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs, getAlertThresholdP95LatencyMs,
getAgentBudgetLlmCostEnabled,
getAgentCostAwareTierEnabled,
getAgentSupervisorFastPathEnabled,
getAgentToolSandboxEnabled,
getEmbeddingModel, getEmbeddingModel,
getEmbeddingProvider, getEmbeddingProvider,
getLongTermSummaryEnabled, getLongTermSummaryEnabled,
getMemoryEvictionEnabled, getMemoryEvictionEnabled,
getNatsAclMode,
getPhoenixLogPrompts, getPhoenixLogPrompts,
getPhoenixOtlpEndpoint, getPhoenixOtlpEndpoint,
getRagHybridEnabled, getRagHybridEnabled,
@@ -17,7 +22,9 @@ import {
getRagEnabled, getRagEnabled,
getSlackAlertWebhookUrl, getSlackAlertWebhookUrl,
getSlackWebhookTimeoutMs, getSlackWebhookTimeoutMs,
getSupervisorFastPathConfidence,
getSupervisorLlmResponseEnabled, getSupervisorLlmResponseEnabled,
getToolAuditMaxLen,
} from "./config.js"; } from "./config.js";
const ENV_KEYS = [ const ENV_KEYS = [
@@ -35,6 +42,13 @@ const ENV_KEYS = [
"AGENT_RAG_RERANKER_TOP_K", "AGENT_RAG_RERANKER_TOP_K",
"AGENT_LONG_TERM_SUMMARY_ENABLED", "AGENT_LONG_TERM_SUMMARY_ENABLED",
"AGENT_MEMORY_EVICTION_ENABLED", "AGENT_MEMORY_EVICTION_ENABLED",
"AGENT_BUDGET_LLM_COST_ENABLED",
"AGENT_COST_AWARE_TIER_ENABLED",
"AGENT_SUPERVISOR_FAST_PATH_ENABLED",
"SUPERVISOR_FAST_PATH_CONFIDENCE",
"AGENT_TOOL_SANDBOX_ENABLED",
"AGENT_TOOL_AUDIT_MAX_LEN",
"AGENT_NATS_ACL_MODE",
"EMBEDDING_MODEL", "EMBEDDING_MODEL",
"EMBEDDING_PROVIDER", "EMBEDDING_PROVIDER",
"PHOENIX_LOG_PROMPTS", "PHOENIX_LOG_PROMPTS",
@@ -129,4 +143,30 @@ describe("config", () => {
expect(getSupervisorLlmResponseEnabled()).toBe(false); expect(getSupervisorLlmResponseEnabled()).toBe(false);
}); });
it("reads operational excellence guardrail configuration", () => {
expect(getAgentBudgetLlmCostEnabled()).toBe(true);
expect(getAgentCostAwareTierEnabled()).toBe(true);
expect(getAgentSupervisorFastPathEnabled()).toBe(true);
expect(getSupervisorFastPathConfidence()).toBe(0.85);
expect(getAgentToolSandboxEnabled()).toBe(false);
expect(getToolAuditMaxLen()).toBe(100_000);
expect(getNatsAclMode()).toBe("log-only");
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
vi.stubEnv("AGENT_COST_AWARE_TIER_ENABLED", "false");
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
vi.stubEnv("SUPERVISOR_FAST_PATH_CONFIDENCE", "0.9");
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
vi.stubEnv("AGENT_TOOL_AUDIT_MAX_LEN", "42");
vi.stubEnv("AGENT_NATS_ACL_MODE", "enforce");
expect(getAgentBudgetLlmCostEnabled()).toBe(false);
expect(getAgentCostAwareTierEnabled()).toBe(false);
expect(getAgentSupervisorFastPathEnabled()).toBe(false);
expect(getSupervisorFastPathConfidence()).toBe(0.9);
expect(getAgentToolSandboxEnabled()).toBe(true);
expect(getToolAuditMaxLen()).toBe(42);
expect(getNatsAclMode()).toBe("enforce");
});
}); });
+32
View File
@@ -231,6 +231,38 @@ export function getAgentBudgetMaxCostUsd(): number {
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5); return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
} }
export function getAgentBudgetLlmCostEnabled(): boolean {
return getBoolean("AGENT_BUDGET_LLM_COST_ENABLED", true);
}
export function getAgentCostAwareTierEnabled(): boolean {
return getBoolean("AGENT_COST_AWARE_TIER_ENABLED", true);
}
export function getAgentSupervisorFastPathEnabled(): boolean {
return getBoolean("AGENT_SUPERVISOR_FAST_PATH_ENABLED", true);
}
export function getSupervisorFastPathConfidence(): number {
return getRatioNumber("SUPERVISOR_FAST_PATH_CONFIDENCE", 0.85);
}
export function getAgentToolSandboxEnabled(): boolean {
return getBoolean("AGENT_TOOL_SANDBOX_ENABLED", false);
}
export function getToolAuditMaxLen(): number {
return getPositiveNumber("AGENT_TOOL_AUDIT_MAX_LEN", 100_000);
}
export function getNatsAclMode(): "off" | "log-only" | "enforce" {
const raw = process.env["AGENT_NATS_ACL_MODE"]?.trim().toLowerCase();
if (raw === "off" || raw === "log-only" || raw === "enforce") {
return raw;
}
return "log-only";
}
export function getAgentRuntimeVerifierEnabled(): boolean { export function getAgentRuntimeVerifierEnabled(): boolean {
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false); return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
} }
+101
View File
@@ -25,6 +25,8 @@ vi.mock("@obs/spans", () => ({
})); }));
import { LlmClient } from "./llm-client.js"; import { LlmClient } from "./llm-client.js";
import { BudgetTracker } from "@agent/budget";
import { runWithContext } from "@obs/context";
import type { ChatCompletionResponse } from "./types.js"; import type { ChatCompletionResponse } from "./types.js";
function okResponse( function okResponse(
@@ -168,4 +170,103 @@ describe("LlmClient endpoint normalization", () => {
}), }),
); );
}); });
it("records LLM cost into the request budget", async () => {
vi.stubGlobal(
"fetch",
vi.fn(() =>
Promise.resolve(
okResponse("reply", {
usage: {
completion_tokens: 50,
prompt_tokens: 100,
total_tokens: 150,
},
}),
),
),
);
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 1,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 5,
});
const client = new LlmClient({
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
providerApiKeys: { openrouter: "test-key" },
});
await runWithContext(
{
budget,
chatroomId: "room-1",
requestId: "req-1",
skill: "onboarding",
startedAt: "2026-05-18T00:00:00.000Z",
trigger: "user-message",
userId: "user-1",
},
() =>
client.chatCompletion([{ content: "hi", role: "user" }], {
modelOverride: "gpt-5.3-codex-spark",
}),
);
expect(budget.snapshot().totalLlmCostUsd).toBe(0.000045);
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
expect.objectContaining({
"budget.llm_cost_usd": 0.000045,
"budget.total_cost_usd": 0.000045,
}),
);
});
it("does not record LLM cost when budget cost tracking is disabled", async () => {
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
vi.stubGlobal(
"fetch",
vi.fn(() =>
Promise.resolve(
okResponse("reply", {
usage: {
completion_tokens: 50,
prompt_tokens: 100,
total_tokens: 150,
},
}),
),
),
);
const budget = new BudgetTracker({
enabled: true,
maxCostUsd: 1,
maxHandoffs: 5,
maxSameAgentVisits: 5,
maxToolCalls: 5,
});
const client = new LlmClient({
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
providerApiKeys: { openrouter: "test-key" },
});
await runWithContext(
{
budget,
chatroomId: "room-1",
requestId: "req-1",
skill: "onboarding",
startedAt: "2026-05-18T00:00:00.000Z",
trigger: "user-message",
userId: "user-1",
},
() =>
client.chatCompletion([{ content: "hi", role: "user" }], {
modelOverride: "gpt-5.3-codex-spark",
}),
);
expect(budget.snapshot().totalLlmCostUsd).toBe(0);
});
}); });
+12
View File
@@ -10,6 +10,7 @@ import type {
import { getContext, getRequestLogger } from "@obs/context"; import { getContext, getRequestLogger } from "@obs/context";
import { maskTraceContent, withLlmSpan } from "@obs/spans"; import { maskTraceContent, withLlmSpan } from "@obs/spans";
import { import {
getAgentBudgetLlmCostEnabled,
getLlmHealthTimeoutMs, getLlmHealthTimeoutMs,
getLlmRequestTimeoutMs, getLlmRequestTimeoutMs,
getLlmTemperature, getLlmTemperature,
@@ -257,11 +258,22 @@ export class LlmClient {
model: request.model, model: request.model,
skill: getContext()?.skill, skill: getContext()?.skill,
}); });
if (getAgentBudgetLlmCostEnabled()) {
getContext()?.budget?.recordLlmCost(costUsd);
}
const budgetSnapshot = getContext()?.budget?.snapshot();
span.setAttributes({ span.setAttributes({
...(shouldLogPrompts ...(shouldLogPrompts
? buildCompletionTraceAttributes(result) ? buildCompletionTraceAttributes(result)
: {}), : {}),
...(budgetSnapshot
? {
"budget.llm_cost_usd": budgetSnapshot.totalLlmCostUsd,
"budget.tool_cost_usd": budgetSnapshot.totalToolCostUsd,
"budget.total_cost_usd": budgetSnapshot.totalCostUsd,
}
: {}),
"gen_ai.response.finish_reasons": payload.choices "gen_ai.response.finish_reasons": payload.choices
?.map((choice) => choice.finish_reason ?? "unknown") ?.map((choice) => choice.finish_reason ?? "unknown")
.filter((reason): reason is string => Boolean(reason)), .filter((reason): reason is string => Boolean(reason)),
+2 -2
View File
@@ -3,12 +3,12 @@ import { AsyncLocalStorage } from 'node:async_hooks';
import type { Logger } from 'pino'; import type { Logger } from 'pino';
import { getLogger } from './logger.js'; import { getLogger } from './logger.js';
import type { BudgetSnapshot } from '@agent/budget'; import type { BudgetTracker } from '@agent/budget';
export interface RequestContext { export interface RequestContext {
chatroomId: string; chatroomId: string;
deadlineAt?: string; deadlineAt?: string;
budget?: BudgetSnapshot | undefined; budget?: BudgetTracker | undefined;
handoffChain?: string[] | undefined; handoffChain?: string[] | undefined;
handoffDepth?: number | undefined; handoffDepth?: number | undefined;
logger: Logger; logger: Logger;
+12
View File
@@ -15,9 +15,21 @@ type RequestBindings = {
function buildRedactionPaths(): string[] { function buildRedactionPaths(): string[] {
const commonPaths = [ const commonPaths = [
'authorization', 'authorization',
'apiKey',
'api_key',
'headers.authorization', 'headers.authorization',
'headers.x-api-key',
'headers["x-api-key"]',
'req.headers.authorization', 'req.headers.authorization',
'req.headers.x-api-key',
'req.headers["x-api-key"]',
'request.headers.authorization', 'request.headers.authorization',
'request.headers.x-api-key',
'request.headers["x-api-key"]',
'request.apiKey',
'request.api_key',
'response.apiKey',
'response.api_key',
]; ];
const secretPaths = KNOWN_SECRET_KEYS.flatMap((key) => [ const secretPaths = KNOWN_SECRET_KEYS.flatMap((key) => [
+23
View File
@@ -4,6 +4,8 @@ import {
formatReviewList, formatReviewList,
formatReviewShow, formatReviewShow,
formatReviewStats, formatReviewStats,
formatToolAuditList,
formatToolAuditShow,
} from "./cli.js"; } from "./cli.js";
describe("operations review CLI formatters", () => { describe("operations review CLI formatters", () => {
@@ -81,4 +83,25 @@ describe("operations review CLI formatters", () => {
}), }),
).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2"); ).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2");
}); });
it("formats tool audit rows", () => {
const entry = {
argsHash: "abc123",
calledAt: "2026-05-18T00:00:00.000Z",
chatroomId: "room-1",
durationMs: 12,
id: "1-0",
ok: true,
resultSummary: "{\"ok\":true}",
skill: "onboarding",
toolName: "validate-nickname",
type: "tool-call" as const,
userId: "user-1",
};
expect(formatToolAuditList([entry])).toContain(
"1-0\t2026-05-18T00:00:00.000Z\tonboarding\tvalidate-nickname\tyes\t12",
);
expect(formatToolAuditShow(entry)).toContain("argsHash: abc123");
});
}); });
+99 -1
View File
@@ -15,6 +15,10 @@ import {
type SupervisorReviewResolution, type SupervisorReviewResolution,
type SupervisorReviewStats, type SupervisorReviewStats,
} from "@skills/supervisor/review-queue"; } from "@skills/supervisor/review-queue";
import {
ToolAuditLogger,
type ListedToolAuditEntry,
} from "@tools/audit";
import { import {
getFlagString, getFlagString,
hasFlag, hasFlag,
@@ -164,6 +168,46 @@ export function formatReviewStats(stats: SupervisorReviewStats): string {
].join("\n"); ].join("\n");
} }
export function formatToolAuditList(items: ListedToolAuditEntry[]): string {
const rows = [
[
"id",
"calledAt",
"skill",
"toolName",
"ok",
"durationMs",
"errorKind",
],
...items.map((item) => [
item.id,
item.calledAt,
item.skill,
item.toolName,
item.ok ? "yes" : "no",
String(item.durationMs),
item.errorKind ?? "-",
]),
];
return rows.map((row) => row.join("\t")).join("\n");
}
export function formatToolAuditShow(item: ListedToolAuditEntry): string {
return [
`id: ${item.id}`,
`calledAt: ${item.calledAt}`,
`skill: ${item.skill}`,
`toolName: ${item.toolName}`,
`chatroomId: ${item.chatroomId}`,
`userId: ${item.userId}`,
`ok: ${item.ok ? "yes" : "no"}`,
`durationMs: ${item.durationMs}`,
`errorKind: ${item.errorKind ?? "-"}`,
`argsHash: ${item.argsHash}`,
`resultSummary: ${item.resultSummary || "-"}`,
].join("\n");
}
async function handleReviewList( async function handleReviewList(
flags: Map<string, string | true>, flags: Map<string, string | true>,
): Promise<void> { ): Promise<void> {
@@ -277,6 +321,52 @@ async function handleReviewStats(
} }
} }
async function handleToolAuditList(
flags: Map<string, string | true>,
): Promise<void> {
const connection = await connectGarnetForCli();
try {
const items = await new ToolAuditLogger(connection.garnetClient).list({
limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? 50,
since: sinceDurationToIso(getFlagString(flags, "since") ?? "24h"),
skill: getFlagString(flags, "skill"),
});
console.info(
hasFlag(flags, "json")
? JSON.stringify(items, null, 2)
: formatToolAuditList(items),
);
} finally {
await connection.close();
}
}
async function handleToolAuditShow(
id: string | undefined,
flags: Map<string, string | true>,
): Promise<void> {
if (!id?.trim()) {
throw new Error("tool-audit:show requires an entry id");
}
const connection = await connectGarnetForCli();
try {
const item = await new ToolAuditLogger(connection.garnetClient).get(id);
if (!item) {
process.exitCode = 2;
console.error(`[tool-audit] entry not found: ${id}`);
return;
}
console.info(
hasFlag(flags, "json")
? JSON.stringify(item, null, 2)
: formatToolAuditShow(item),
);
} finally {
await connection.close();
}
}
async function main(): Promise<void> { async function main(): Promise<void> {
loadEnvironmentFromNodeEnv(); loadEnvironmentFromNodeEnv();
initLogger(); initLogger();
@@ -298,9 +388,17 @@ async function main(): Promise<void> {
await handleReviewStats(flags); await handleReviewStats(flags);
return; return;
} }
if (command === "tool-audit:list") {
await handleToolAuditList(flags);
return;
}
if (command === "tool-audit:show") {
await handleToolAuditShow(positionals[0], flags);
return;
}
throw new Error( throw new Error(
"Unknown command. Use review:list, review:show, review:resolve, or review:stats.", "Unknown command. Use review:list, review:show, review:resolve, review:stats, tool-audit:list, or tool-audit:show.",
); );
} }
+3
View File
@@ -46,6 +46,9 @@ describe("escalation trigger", () => {
maxHandoffs: 4, maxHandoffs: 4,
maxSameAgentVisits: 2, maxSameAgentVisits: 2,
maxToolCalls: 20, maxToolCalls: 20,
totalCostUsd: 0,
totalLlmCostUsd: 0,
totalToolCostUsd: 0,
toolCalls: 0, toolCalls: 0,
visits: {}, visits: {},
}, },
+1
View File
@@ -16,5 +16,6 @@ export function createOnboardingSkill(deps: SkillDeps): Skill {
deps.shadowEvalBus, deps.shadowEvalBus,
deps.natsClient, deps.natsClient,
deps.memoryService, deps.memoryService,
deps.escalationHistoryService,
); );
} }
+45 -2
View File
@@ -102,9 +102,9 @@ describe("decidePreGenerationRouting", () => {
}, },
{ {
confidence: 0.3, confidence: 0.3,
expected: { kind: "tier0-only" }, expected: { kind: "tier1-direct", skipPlanner: false },
intentKind: "off-topic-to-other" as const, intentKind: "off-topic-to-other" as const,
name: "keeps low-confidence off-topic on tier0", name: "promotes low-confidence off-topic to tier1",
phase: { kind: "fresh" } as OnboardingPhase, phase: { kind: "fresh" } as OnboardingPhase,
}, },
{ {
@@ -319,6 +319,49 @@ describe("decidePreGenerationRouting", () => {
).toEqual(expected); ).toEqual(expected);
}); });
it("keeps high-confidence low-risk requests on tier0", () => {
expect(
decidePreGenerationRouting({
classified: classified("age-question", 0.95),
derived: derived({ kind: "deadline-near", remaining: ["photo"] }, intent("age-question"), {
tier: 1,
}),
intentRiskTier: "low",
request: request(),
session: session(),
userEscalationCount: 0,
}),
).toEqual({ kind: "tier0-only" });
});
it("promotes low-confidence high-risk requests with escalation history", () => {
expect(
decidePreGenerationRouting({
classified: classified("unknown", 0.49),
derived: derived({ kind: "fresh" }, intent("unknown")),
intentRiskTier: "high",
request: request(),
session: session(),
userEscalationCount: 4,
}),
).toEqual({ kind: "tier1-direct", skipPlanner: false });
});
it("prefers tier0 during latency degradation unless safety already promoted", () => {
expect(
decidePreGenerationRouting({
classified: classified("age-question", 0.8),
derived: derived({ kind: "deadline-near", remaining: ["nickname"] }, intent("age-question"), {
tier: 1,
}),
intentRiskTier: "medium",
recentLatencyP95Ms: 10_000,
request: request(),
session: session(),
}),
).toEqual({ kind: "tier0-only" });
});
it("trains, saves, loads, and applies a learned router model", () => { it("trains, saves, loads, and applies a learned router model", () => {
const lowConfidenceInput = { const lowConfidenceInput = {
classified: classified("age-question", 0.25), classified: classified("age-question", 0.25),
+49
View File
@@ -3,6 +3,10 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname } from "node:path"; import { dirname } from "node:path";
import type { AgentRequest } from "@agent/types"; import type { AgentRequest } from "@agent/types";
import {
getAgentCostAwareTierEnabled,
getAlertThresholdP95LatencyMs,
} from "../../../config.js";
import type { import type {
ClassifiedIntent, ClassifiedIntent,
DerivedState, DerivedState,
@@ -58,8 +62,11 @@ export interface RouterTrainingSample {
export interface RouterInput { export interface RouterInput {
classified: ClassifiedIntent; classified: ClassifiedIntent;
derived: DerivedState; derived: DerivedState;
intentRiskTier?: "low" | "medium" | "high" | undefined;
recentLatencyP95Ms?: number | undefined;
request: AgentRequest; request: AgentRequest;
session: OnboardingSession | null; session: OnboardingSession | null;
userEscalationCount?: number | undefined;
} }
export const ROUTER_FEATURE_NAMES = [ export const ROUTER_FEATURE_NAMES = [
@@ -294,6 +301,43 @@ function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
); );
} }
function isLatencyDegraded(input: RouterInput): boolean {
const p95 = input.recentLatencyP95Ms;
return (
typeof p95 === "number" &&
Number.isFinite(p95) &&
p95 > getAlertThresholdP95LatencyMs("onboarding")
);
}
function maybeDecideCostAwareRouting(input: RouterInput): RoutingDecision | null {
if (!getAgentCostAwareTierEnabled()) {
return null;
}
const confidence = clamp01(input.classified.confidence);
const riskTier = input.intentRiskTier ?? "medium";
const escalationCount = Math.max(0, input.userEscalationCount ?? 0);
const lowLatency = !isLatencyDegraded(input);
if (confidence < 0.5 || (riskTier === "high" && escalationCount > 3)) {
if (input.derived.responseMode === "llm-with-tools") {
return { kind: "tier1-with-tools" };
}
return { kind: "tier1-direct", skipPlanner: false };
}
if (!lowLatency) {
return { kind: "tier0-only" };
}
if (confidence >= 0.9 && riskTier === "low") {
return { kind: "tier0-only" };
}
return null;
}
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision { export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
const { classified, derived, request, session } = input; const { classified, derived, request, session } = input;
const { confidence, intent } = classified; const { confidence, intent } = classified;
@@ -334,6 +378,11 @@ export function decidePreGenerationRouting(input: RouterInput): RoutingDecision
}; };
} }
const costAwareDecision = maybeDecideCostAwareRouting(input);
if (costAwareDecision) {
return costAwareDecision;
}
if ( if (
phase.kind === "nickname-rejected" || phase.kind === "nickname-rejected" ||
intent.kind === "discord-verify-question" || intent.kind === "discord-verify-question" ||
@@ -142,4 +142,71 @@ describe("runToolPlan", () => {
toolName: "lookup-community-rule", toolName: "lookup-community-rule",
}); });
}); });
it("appends audit entries for successful tool calls", async () => {
const streams: Array<{
fields: Record<string, string>;
key: string;
options?: { maxLen?: number | undefined };
}> = [];
const tool: Tool<{ topic: string }, { text: string }> = {
name: "lookup-community-rule",
description: "규칙 조회",
requires: [],
argsSchema: z.object({
topic: z.string(),
}),
resultSchema: z.object({
text: z.string(),
}),
execute: () => Promise.resolve({ text: "하트 규칙" }),
};
await runToolPlan(
{
composeWith: "all-then-llm",
steps: [
{
id: "rule",
tool: "lookup-community-rule",
argsLiteral: {
topic: "heart",
},
},
],
},
[tool],
{
garnetClient: {
appendStream: (
key: string,
fields: Record<string, string>,
options?: { maxLen?: number | undefined },
) => {
streams.push({
fields,
key,
...(options ? { options } : {}),
});
return Promise.resolve("1-0");
},
} as never,
},
request(),
);
expect(streams[0]).toMatchObject({
key: "lolympus-agent:tool-calls",
fields: {
ok: "true",
skill: "onboarding",
toolName: "lookup-community-rule",
type: "tool-call",
},
options: {
maxLen: 100_000,
},
});
expect(streams[0]?.fields["argsHash"]).toEqual(expect.any(String));
});
}); });
@@ -1,5 +1,8 @@
import type { AgentRequest } from "@agent/types"; import type { AgentRequest } from "@agent/types";
import { getRequestLogger } from "@obs/context";
import { withToolSpan } from "@obs/spans"; import { withToolSpan } from "@obs/spans";
import { hashToolArgs, ToolAuditLogger } from "@tools/audit";
import { buildScopedContext } from "@tools/scoped-context";
import type { import type {
Tool, Tool,
ToolContext, ToolContext,
@@ -208,6 +211,41 @@ function makePlanFailure(message: string, details?: unknown): ToolExecutionFailu
}); });
} }
async function appendToolAudit(input: {
args: unknown;
ctx: ToolContext;
durationMs: number;
errorKind?: string | undefined;
ok: boolean;
request: AgentRequest;
resultSummary: string;
toolName: string;
}): Promise<void> {
if (!input.ctx.garnetClient) {
return;
}
try {
await new ToolAuditLogger(input.ctx.garnetClient).append({
type: "tool-call",
argsHash: hashToolArgs(input.args),
calledAt: new Date().toISOString(),
chatroomId: input.request.chatroomId,
durationMs: input.durationMs,
...(input.errorKind ? { errorKind: input.errorKind } : {}),
ok: input.ok,
resultSummary: input.resultSummary,
skill: input.request.skill,
toolName: input.toolName,
userId: input.request.userId,
});
} catch (error) {
getRequestLogger().warn(
{ err: error, toolName: input.toolName },
"[ToolAudit] Failed to append tool audit entry",
);
}
}
function orderPlan(plan: ToolPlan): OrderedPlan { function orderPlan(plan: ToolPlan): OrderedPlan {
const failures: ToolExecutionFailure[] = []; const failures: ToolExecutionFailure[] = [];
const byId = new Map<string, ToolPlanStep>(); const byId = new Map<string, ToolPlanStep>();
@@ -278,19 +316,31 @@ async function runStep(input: {
const traceStep = getActiveProcessTrace()?.startStep("tool-call"); const traceStep = getActiveProcessTrace()?.startStep("tool-call");
let args: Record<string, unknown> = {}; let args: Record<string, unknown> = {};
const startedAt = Date.now();
try { try {
args = buildArgs(input); args = buildArgs(input);
const validated = tool.argsSchema.parse(args); const validated = tool.argsSchema.parse(args);
const scopedContext = buildScopedContext(tool.requires, input.ctx, tool.name);
const result = await withToolSpan( const result = await withToolSpan(
tool.name, tool.name,
{ {
"tool.name": tool.name, "tool.name": tool.name,
"tool.plan.step_id": input.step.id, "tool.plan.step_id": input.step.id,
}, },
() => tool.execute(validated, input.ctx), () => tool.execute(validated, scopedContext),
); );
const structuredFailure = getStructuredFailure(input.step, result); const structuredFailure = getStructuredFailure(input.step, result);
if (structuredFailure) { if (structuredFailure) {
await appendToolAudit({
args,
ctx: input.ctx,
durationMs: Math.max(0, Date.now() - startedAt),
errorKind: structuredFailure.kind,
ok: false,
request: input.request,
resultSummary: structuredFailure.message,
toolName: tool.name,
});
traceStep?.finish({ traceStep?.finish({
args, args,
errorKind: structuredFailure.kind, errorKind: structuredFailure.kind,
@@ -302,15 +352,35 @@ async function runStep(input: {
} }
input.results[input.step.id] = result; input.results[input.step.id] = result;
const resultSummary = summarizeToolResult(result);
await appendToolAudit({
args,
ctx: input.ctx,
durationMs: Math.max(0, Date.now() - startedAt),
ok: true,
request: input.request,
resultSummary,
toolName: tool.name,
});
traceStep?.finish({ traceStep?.finish({
args, args,
name: tool.name, name: tool.name,
ok: true, ok: true,
resultSummary: summarizeToolResult(result), resultSummary,
}); });
return null; return null;
} catch (error) { } catch (error) {
const failure = classifyError(tool.name, error); const failure = classifyError(tool.name, error);
await appendToolAudit({
args,
ctx: input.ctx,
durationMs: Math.max(0, Date.now() - startedAt),
errorKind: failure.kind,
ok: false,
request: input.request,
resultSummary: failure.message,
toolName: tool.name,
});
traceStep?.finish({ traceStep?.finish({
args, args,
errorKind: failure.kind, errorKind: failure.kind,
+26
View File
@@ -1,6 +1,7 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types'; import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
import type { GarnetClient } from '@common/garnet'; import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats'; import type { NatsClient } from '@common/nats';
import type { EscalationHistoryService } from '../../memory/escalation-history.js';
import type { MemoryService } from '../../memory/memory-types.js'; import type { MemoryService } from '../../memory/memory-types.js';
import { import {
buildLongTermSummary, buildLongTermSummary,
@@ -118,6 +119,7 @@ export class OnboardingSkill implements Skill {
shadowEvalPublisher?: ShadowEvalPublisher, shadowEvalPublisher?: ShadowEvalPublisher,
private readonly natsClient?: Pick<NatsClient, 'request'>, private readonly natsClient?: Pick<NatsClient, 'request'>,
private readonly memoryService?: MemoryService, private readonly memoryService?: MemoryService,
private readonly escalationHistoryService?: EscalationHistoryService,
) { ) {
this.classifier = new OnboardingIntentClassifier(llmClient); this.classifier = new OnboardingIntentClassifier(llmClient);
this.blackboardStore = new OnboardingBlackboardStore( this.blackboardStore = new OnboardingBlackboardStore(
@@ -186,13 +188,17 @@ export class OnboardingSkill implements Skill {
const deriveStep = trace?.startStep('derive'); const deriveStep = trace?.startStep('derive');
const derived = deriveState(session, request.trigger, classified.intent); const derived = deriveState(session, request.trigger, classified.intent);
const escalationHistory = await this.loadEscalationHistory(request.userId);
const routerMode = getAgentRouterMode(); const routerMode = getAgentRouterMode();
if (routerMode === 'static' || routerMode === 'learned-v1') { if (routerMode === 'static' || routerMode === 'learned-v1') {
const routerInput = { const routerInput = {
classified, classified,
derived, derived,
intentRiskTier: this.capabilityCard().riskTier,
recentLatencyP95Ms: this.getRecentLatencyP95Ms(request),
request, request,
session, session,
userEscalationCount: escalationHistory?.totalEscalations ?? 0,
}; };
derived.routingDecision = derived.routingDecision =
routerMode === 'learned-v1' routerMode === 'learned-v1'
@@ -353,6 +359,26 @@ export class OnboardingSkill implements Skill {
return normalizeOnboardingSession(rawSession); return normalizeOnboardingSession(rawSession);
} }
private async loadEscalationHistory(userId: string) {
if (!this.escalationHistoryService) {
return null;
}
try {
return await this.escalationHistoryService.load(userId);
} catch (error) {
getRequestLogger().warn(
{ err: error },
'[OnboardingSkill] Failed to load escalation history',
);
return null;
}
}
private getRecentLatencyP95Ms(request: AgentRequest): number | undefined {
const raw = request.context?.['recentLatencyP95Ms'];
return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined;
}
private async decideEarlyAbstention( private async decideEarlyAbstention(
request: AgentRequest, request: AgentRequest,
classified: ClassifiedIntent, classified: ClassifiedIntent,
+43 -7
View File
@@ -1,17 +1,53 @@
import type { HandoffPacket } from "@agent/handoff-packet"; import type { HandoffPacket } from "@agent/handoff-packet";
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null { export interface SupervisorRuleRoute {
const text = [ confidence: number;
target: string | null;
}
const RULES = [
{
pattern: /(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu,
target: "moderation",
},
{
pattern: /(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu,
target: "onboarding",
},
] as const;
function packetText(packet: HandoffPacket | undefined): string {
return [
packet?.userGoal, packet?.userGoal,
packet?.summary, packet?.summary,
packet?.handoffReason, packet?.handoffReason,
].join(" "); ].join(" ");
}
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) { export function routeSupervisorByRules(
return "moderation"; packet: HandoffPacket | undefined,
): SupervisorRuleRoute {
const text = packetText(packet);
const matches = RULES.filter((rule) => rule.pattern.test(text));
if (matches.length === 1) {
return {
confidence: 0.9,
target: matches[0]?.target ?? null,
};
} }
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) { if (matches.length > 1) {
return "onboarding"; return {
confidence: 0.6,
target: matches[0]?.target ?? null,
};
} }
return null; return {
confidence: 0,
target: null,
};
}
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
return routeSupervisorByRules(packet).target;
} }
+41 -1
View File
@@ -119,7 +119,7 @@ describe("SupervisorSkill", () => {
expect(response.text).toContain("운영자"); expect(response.text).toContain("운영자");
expect(response.metadata).toMatchObject({ expect(response.metadata).toMatchObject({
routedBy: "rules", routedBy: "regex-fast-path",
source: "template", source: "template",
supervisorReason: "max-hop", supervisorReason: "max-hop",
suggestedTarget: "onboarding", suggestedTarget: "onboarding",
@@ -149,6 +149,7 @@ describe("SupervisorSkill", () => {
}); });
it("uses LLM routing when the decision is confident and valid", async () => { it("uses LLM routing when the decision is confident and valid", async () => {
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
const garnet = new FakeGarnet(); const garnet = new FakeGarnet();
const nats = new FakeNats(); const nats = new FakeNats();
const skill = new SupervisorSkill( const skill = new SupervisorSkill(
@@ -189,6 +190,45 @@ describe("SupervisorSkill", () => {
}); });
}); });
it("uses regex fast-path before calling the supervisor LLM", async () => {
const garnet = new FakeGarnet();
const nats = new FakeNats();
const llm = {
chatCompletion: vi.fn(),
};
const skill = new SupervisorSkill(
garnet,
nats,
llm as unknown as Pick<LlmClient, "chatCompletion">,
registry,
);
const packet = buildHandoffPacket({
handoff: {
confidence: 0.7,
evidenceRefs: [],
expectedOutput: "text",
reason: "out-of-scope",
risk: "medium",
targetAgent: "supervisor",
taskSpec: "광고 규칙 문의",
},
request: baseRequest,
response: sourceResponse,
sourceChain: ["onboarding", "supervisor"],
});
const response = await skill.handle(
{ ...baseRequest, context: { handoffPacket: packet } },
[],
);
expect(llm.chatCompletion).not.toHaveBeenCalled();
expect(response.metadata).toMatchObject({
routedBy: "regex-fast-path",
suggestedTarget: "moderation",
});
});
it("uses safe LLM userMessage for supervisor replies", async () => { it("uses safe LLM userMessage for supervisor replies", async () => {
const garnet = new FakeGarnet(); const garnet = new FakeGarnet();
const nats = new FakeNats(); const nats = new FakeNats();
+29 -5
View File
@@ -3,6 +3,7 @@ import type {
AgentResponse, AgentResponse,
ConversationMessage, ConversationMessage,
} from "@agent/types"; } from "@agent/types";
import { trace } from "@opentelemetry/api";
import { readHandoffPacket } from "@agent/handoff-packet"; import { readHandoffPacket } from "@agent/handoff-packet";
import type { HandoffPacket } from "@agent/handoff-packet"; import type { HandoffPacket } from "@agent/handoff-packet";
import type { GarnetClient } from "@common/garnet"; import type { GarnetClient } from "@common/garnet";
@@ -19,6 +20,8 @@ import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader"; import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
import { getLogger } from "@obs/logger"; import { getLogger } from "@obs/logger";
import { import {
getAgentSupervisorFastPathEnabled,
getSupervisorFastPathConfidence,
getSupervisorLlmConfidenceThreshold, getSupervisorLlmConfidenceThreshold,
getSupervisorLlmEnabled, getSupervisorLlmEnabled,
getSupervisorLlmResponseEnabled, getSupervisorLlmResponseEnabled,
@@ -28,7 +31,7 @@ import {
buildSupervisorUserPrompt, buildSupervisorUserPrompt,
} from "./capability-prompt.js"; } from "./capability-prompt.js";
import { appendSupervisorReview } from "./review-queue.js"; import { appendSupervisorReview } from "./review-queue.js";
import { routeSupervisorTarget } from "./routing-rules.js"; import { routeSupervisorByRules } from "./routing-rules.js";
import { import {
supervisorRoutingSchema, supervisorRoutingSchema,
type SupervisorRoutingDecision, type SupervisorRoutingDecision,
@@ -114,13 +117,30 @@ export class SupervisorSkill implements Skill {
): Promise<AgentResponse> { ): Promise<AgentResponse> {
const packet = readHandoffPacket(request); const packet = readHandoffPacket(request);
const escalationHistory = await this.loadEscalationHistory(request.userId); const escalationHistory = await this.loadEscalationHistory(request.userId);
const llmDecision = await this.routeByLlm(packet, escalationHistory); const rulesDecision = routeSupervisorByRules(packet);
const useRegexFastPath =
getAgentSupervisorFastPathEnabled() &&
rulesDecision.target !== null &&
rulesDecision.confidence >= getSupervisorFastPathConfidence() &&
this.isKnownTarget(rulesDecision.target);
const llmDecision = useRegexFastPath
? null
: await this.routeByLlm(packet, escalationHistory);
const useLlmRoute = const useLlmRoute =
llmDecision !== null && llmDecision !== null &&
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold(); llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
const target = useLlmRoute const target = useLlmRoute
? llmDecision.target ? llmDecision.target
: routeSupervisorTarget(packet); : rulesDecision.target;
const routedBy = useRegexFastPath
? "regex-fast-path"
: useLlmRoute
? "llm"
: "rules-fallback";
trace.getActiveSpan()?.setAttributes({
"supervisor.routed_by": routedBy,
"supervisor.rules_confidence": rulesDecision.confidence,
});
const reason = packet?.handoffReason ?? "out-of-scope"; const reason = packet?.handoffReason ?? "out-of-scope";
const queuedAt = new Date().toISOString(); const queuedAt = new Date().toISOString();
@@ -169,7 +189,8 @@ export class SupervisorSkill implements Skill {
return { return {
confidence: 0.6, confidence: 0.6,
metadata: { metadata: {
routedBy: useLlmRoute ? "llm" : "rules", routedBy,
rulesConfidence: rulesDecision.confidence,
source: userMessage.source, source: userMessage.source,
supervisorReason: useLlmRoute ? llmDecision.reason : reason, supervisorReason: useLlmRoute ? llmDecision.reason : reason,
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0, supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
@@ -234,7 +255,10 @@ export class SupervisorSkill implements Skill {
private isKnownTarget(target: string): boolean { private isKnownTarget(target: string): boolean {
return ( return (
target === "human-review" || Boolean(this.skillRegistry?.get(target)) target === "human-review" ||
target === "onboarding" ||
target === "moderation" ||
Boolean(this.skillRegistry?.get(target))
); );
} }
+135
View File
@@ -0,0 +1,135 @@
import { createHash } from "node:crypto";
import type { GarnetClient } from "@common/garnet";
import { getToolAuditMaxLen } from "../config.js";
export const TOOL_AUDIT_STREAM = "lolympus-agent:tool-calls";
export interface ToolAuditEntry {
type: "tool-call";
toolName: string;
chatroomId: string;
userId: string;
argsHash: string;
resultSummary: string;
ok: boolean;
errorKind?: string | undefined;
durationMs: number;
skill: string;
traceId?: string | undefined;
calledAt: string;
}
export interface ListedToolAuditEntry extends ToolAuditEntry {
id: string;
}
function normalizeForHash(value: unknown): unknown {
if (Array.isArray(value)) {
return value.map(normalizeForHash);
}
if (typeof value !== "object" || value === null) {
return value;
}
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.filter(([key]) => !/(password|secret|token|api[_-]?key|authorization)/iu.test(key))
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, entry]) => [key, normalizeForHash(entry)]),
);
}
function asString(value: string | undefined): string | undefined {
return value?.trim() ? value : undefined;
}
function parseAuditEntry(
id: string,
fields: Record<string, string>,
): ListedToolAuditEntry | null {
if (fields["type"] !== "tool-call") {
return null;
}
return {
id,
type: "tool-call",
argsHash: fields["argsHash"] ?? "",
calledAt: fields["calledAt"] ?? "",
chatroomId: fields["chatroomId"] ?? "",
durationMs: Number(fields["durationMs"] ?? 0),
...(asString(fields["errorKind"]) ? { errorKind: fields["errorKind"] } : {}),
ok: fields["ok"] === "true",
resultSummary: fields["resultSummary"] ?? "",
skill: fields["skill"] ?? "",
...(asString(fields["traceId"]) ? { traceId: fields["traceId"] } : {}),
toolName: fields["toolName"] ?? "",
userId: fields["userId"] ?? "",
};
}
export function hashToolArgs(args: unknown): string {
return createHash("sha256")
.update(JSON.stringify(normalizeForHash(args)))
.digest("hex")
.slice(0, 16);
}
export class ToolAuditLogger {
constructor(
private readonly garnetClient: Pick<
GarnetClient,
"appendStream" | "readStreamRange"
>,
) {}
async append(entry: ToolAuditEntry): Promise<void> {
await this.garnetClient.appendStream(
TOOL_AUDIT_STREAM,
Object.fromEntries(
Object.entries(entry).map(([key, value]) => [key, String(value)]),
),
{ maxLen: getToolAuditMaxLen() },
);
}
async list(opts: {
skill?: string | undefined;
since?: string | undefined;
limit?: number | undefined;
}): Promise<ListedToolAuditEntry[]> {
const limit = Math.max(1, Math.min(opts.limit ?? 50, 500));
const entries = await this.garnetClient.readStreamRange(
TOOL_AUDIT_STREAM,
"-",
"+",
{ count: limit * 4 },
);
const sinceTime = opts.since ? Date.parse(opts.since) : NaN;
return entries
.map((entry) => parseAuditEntry(entry.id, entry.message))
.filter((entry): entry is ListedToolAuditEntry => entry !== null)
.filter((entry) => !opts.skill || entry.skill === opts.skill)
.filter((entry) => {
if (!Number.isFinite(sinceTime)) {
return true;
}
const calledAtTime = Date.parse(entry.calledAt);
return Number.isFinite(calledAtTime) && calledAtTime >= sinceTime;
})
.slice(-limit)
.reverse();
}
async get(id: string): Promise<ListedToolAuditEntry | null> {
const entries = await this.garnetClient.readStreamRange(
TOOL_AUDIT_STREAM,
id,
id,
{ count: 1 },
);
const first = entries[0];
return first ? parseAuditEntry(first.id, first.message) : null;
}
}
+1
View File
@@ -93,6 +93,7 @@ export const lookupRuleTool: Tool<
name: "lookup-rule", name: "lookup-rule",
description: description:
"광고, 도배, 욕설, 신고, 제재, 이의 제기, 개인정보 같은 moderation 규칙을 조회한다.", "광고, 도배, 욕설, 신고, 제재, 이의 제기, 개인정보 같은 moderation 규칙을 조회한다.",
requires: ["memory-search"],
argsSchema, argsSchema,
resultSchema, resultSchema,
async execute(args, ctx) { async execute(args, ctx) {
@@ -41,6 +41,7 @@ export const checkOnboardingProgressTool: Tool<
name: "check-onboarding-progress", name: "check-onboarding-progress",
description: description:
"현재 유저의 온보딩 진행 상태를 읽어 카카오톡 단계와 디스코드 단계를 포함한 남은 절차를 구조화해서 반환한다.", "현재 유저의 온보딩 진행 상태를 읽어 카카오톡 단계와 디스코드 단계를 포함한 남은 절차를 구조화해서 반환한다.",
requires: ["garnet", "user-session"],
argsSchema, argsSchema,
resultSchema, resultSchema,
async execute(args, ctx) { async execute(args, ctx) {
+1
View File
@@ -24,6 +24,7 @@ export const getRoomContextTool: Tool<
name: "get-room-context", name: "get-room-context",
description: description:
"최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.", "최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.",
requires: ["room-context"],
argsSchema, argsSchema,
resultSchema, resultSchema,
async execute(args, ctx) { async execute(args, ctx) {
@@ -90,6 +90,7 @@ export const lookupCommunityRuleTool: Tool<
name: "lookup-community-rule", name: "lookup-community-rule",
description: description:
"나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 조회한다.", "나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 조회한다.",
requires: ["memory-search"],
argsSchema, argsSchema,
resultSchema, resultSchema,
async execute(args, ctx) { async execute(args, ctx) {
@@ -32,6 +32,7 @@ export const validateNicknameTool: Tool<
name: "validate-nickname", name: "validate-nickname",
description: description:
"새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.", "새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.",
requires: [],
argsSchema, argsSchema,
resultSchema, resultSchema,
execute(args) { execute(args) {
@@ -102,6 +102,7 @@ export const verifyDiscordNicknameTool: Tool<
name: 'verify-discord-nickname', name: 'verify-discord-nickname',
description: description:
'현재 유저의 카카오톡 인증 닉네임과 세션 입장 시각을 기준으로 디스코드 멤버를 찾아, 닉네임 변경이 확인됐는지 구조화해서 반환한다.', '현재 유저의 카카오톡 인증 닉네임과 세션 입장 시각을 기준으로 디스코드 멤버를 찾아, 닉네임 변경이 확인됐는지 구조화해서 반환한다.',
requires: ['garnet', 'nats', 'user-session'],
argsSchema, argsSchema,
resultSchema, resultSchema,
async execute(_args, ctx) { async execute(_args, ctx) {
+62
View File
@@ -0,0 +1,62 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildScopedContext } from "./scoped-context.js";
import type { ToolContext } from "./tool.js";
describe("buildScopedContext", () => {
afterEach(() => {
vi.unstubAllEnvs();
});
it("returns the full context when sandbox is disabled", () => {
const ctx: ToolContext = {
chatroomId: "room-1",
garnetClient: {} as ToolContext["garnetClient"],
};
expect(buildScopedContext([], ctx, "pure-tool")).toBe(ctx);
});
it("throws when sandboxed tools access undeclared permissions", () => {
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
const ctx: ToolContext = {
chatroomId: "room-1",
garnetClient: {} as ToolContext["garnetClient"],
userId: "user-1",
};
const scoped = buildScopedContext([], ctx, "validate-nickname");
expect(scoped.chatroomId).toBe("room-1");
expect(() => scoped.garnetClient).toThrow(
"Tool validate-nickname does not declare requires:garnet",
);
});
it("exposes declared memory search without profile access", async () => {
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
const search = vi.fn(() => Promise.resolve([]));
const ctx = {
memoryService: {
longTerm: {
search,
loadUserProfile: vi.fn(),
},
session: {},
assembleWorkingMemory: vi.fn(),
},
} as unknown as ToolContext;
const scoped = buildScopedContext(
["memory-search"],
ctx,
"lookup-community-rule",
);
await scoped.memoryService?.longTerm.search("heart", { topK: 1 });
expect(search).toHaveBeenCalledWith("heart", { topK: 1 });
expect(() => scoped.memoryService?.longTerm.loadUserProfile("user-1")).toThrow(
"Tool lookup-community-rule does not declare requires:memory-profile",
);
});
});
+149
View File
@@ -0,0 +1,149 @@
import { getAgentToolSandboxEnabled } from "../config.js";
import type { LongTermMemory, MemoryService } from "../memory/memory-types.js";
import { getLogger } from "../observability/logger.js";
import type { ToolContext, ToolPermission } from "./tool.js";
const PERMISSION_BY_FIELD: Record<string, ToolPermission> = {
garnetClient: "garnet",
memoryService: "memory-search",
natsClient: "nats",
roomContextService: "room-context",
session: "user-session",
toolRegistry: "tool-registry",
};
function deniedAccess(toolName: string, permission: ToolPermission): never {
throw new Error(`Tool ${toolName} does not declare requires:${permission}`);
}
function defineDeniedGetter<T extends object>(
target: T,
key: string,
toolName: string,
permission: ToolPermission,
): void {
Object.defineProperty(target, key, {
enumerable: true,
get: () => deniedAccess(toolName, permission),
});
}
function buildScopedMemoryService(
requires: ReadonlySet<ToolPermission>,
fullContext: ToolContext,
toolName: string,
): MemoryService | undefined {
if (!fullContext.memoryService) {
return undefined;
}
const longTerm: Partial<LongTermMemory> = {};
if (requires.has("memory-search")) {
longTerm.search = fullContext.memoryService.longTerm.search.bind(
fullContext.memoryService.longTerm,
);
} else {
defineDeniedGetter(longTerm, "search", toolName, "memory-search");
}
if (requires.has("memory-profile")) {
longTerm.loadUserProfile =
fullContext.memoryService.longTerm.loadUserProfile.bind(
fullContext.memoryService.longTerm,
);
} else {
defineDeniedGetter(longTerm, "loadUserProfile", toolName, "memory-profile");
}
defineDeniedGetter(longTerm, "updateUserProfile", toolName, "memory-profile");
defineDeniedGetter(longTerm, "upsertDocument", toolName, "memory-search");
defineDeniedGetter(longTerm, "deleteDocument", toolName, "memory-search");
return {
longTerm: longTerm as LongTermMemory,
session: fullContext.memoryService.session,
assembleWorkingMemory: fullContext.memoryService.assembleWorkingMemory.bind(
fullContext.memoryService,
),
};
}
function findUndeclaredContextPermissions(
requires: ReadonlySet<ToolPermission>,
fullContext: ToolContext,
): ToolPermission[] {
return [
...new Set(
Object.entries(PERMISSION_BY_FIELD)
.filter(
([field, permission]) =>
!requires.has(permission) &&
fullContext[field as keyof ToolContext] !== undefined,
)
.map(([, permission]) => permission),
),
];
}
export function buildScopedContext(
requires: readonly ToolPermission[] | undefined,
fullContext: ToolContext,
toolName = "unknown",
): ToolContext {
const sandboxEnabled = getAgentToolSandboxEnabled();
if (!requires) {
if (sandboxEnabled) {
getLogger().warn(
{ toolName },
"[ToolSandbox] Tool has no requires declaration; using full context",
);
}
return fullContext;
}
if (!sandboxEnabled) {
const undeclared = findUndeclaredContextPermissions(
new Set(requires),
fullContext,
);
if (undeclared.length > 0) {
getLogger().warn(
{ toolName, undeclared },
"[ToolSandbox] Tool context includes undeclared permissions",
);
}
return fullContext;
}
const required = new Set(requires);
const scoped: ToolContext = {
chatroomId: fullContext.chatroomId,
userId: fullContext.userId,
};
if (required.has("garnet")) {
scoped.garnetClient = fullContext.garnetClient;
}
if (required.has("nats")) {
scoped.natsClient = fullContext.natsClient;
}
if (required.has("room-context")) {
scoped.roomContextService = fullContext.roomContextService;
}
if (required.has("tool-registry")) {
scoped.toolRegistry = fullContext.toolRegistry;
}
if (required.has("user-session")) {
scoped.session = fullContext.session;
}
if (required.has("memory-search") || required.has("memory-profile")) {
scoped.memoryService = buildScopedMemoryService(required, fullContext, toolName);
}
for (const [field, permission] of Object.entries(PERMISSION_BY_FIELD)) {
if (field in scoped) {
continue;
}
defineDeniedGetter(scoped, field, toolName, permission);
}
return scoped;
}
+10
View File
@@ -17,6 +17,15 @@ export interface ToolContext {
toolRegistry?: ToolRegistry | undefined; toolRegistry?: ToolRegistry | undefined;
} }
export type ToolPermission =
| "garnet"
| "memory-profile"
| "memory-search"
| "nats"
| "room-context"
| "tool-registry"
| "user-session";
export type ToolErrorKind = export type ToolErrorKind =
| "args-invalid" | "args-invalid"
| "transient" | "transient"
@@ -34,6 +43,7 @@ export interface ToolExecutionFailure {
export interface Tool<A, R> { export interface Tool<A, R> {
name: string; name: string;
description: string; description: string;
requires?: readonly ToolPermission[] | undefined;
argsSchema: ZodType<A>; argsSchema: ZodType<A>;
resultSchema: ZodType<R>; resultSchema: ZodType<R>;
execute(args: A, ctx: ToolContext): Promise<R>; execute(args: A, ctx: ToolContext): Promise<R>;