Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc0f8e0773 | ||
|
|
d011a2829c | ||
|
|
21df14cd00 |
No files matched your search
@@ -36,6 +36,8 @@
|
|||||||
"review:show": "node --import tsx src/operations/cli.ts review:show",
|
"review:show": "node --import tsx src/operations/cli.ts review:show",
|
||||||
"review:resolve": "node --import tsx src/operations/cli.ts review:resolve",
|
"review:resolve": "node --import tsx src/operations/cli.ts review:resolve",
|
||||||
"review:stats": "node --import tsx src/operations/cli.ts review:stats",
|
"review:stats": "node --import tsx src/operations/cli.ts review:stats",
|
||||||
|
"tool-audit:list": "node --import tsx src/operations/cli.ts tool-audit:list",
|
||||||
|
"tool-audit:show": "node --import tsx src/operations/cli.ts tool-audit:show",
|
||||||
"start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"",
|
"start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"",
|
||||||
"start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"",
|
"start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"",
|
||||||
"start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"",
|
"start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"",
|
||||||
|
|||||||
@@ -59,6 +59,30 @@ describe("BudgetTracker", () => {
|
|||||||
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
|
expect(costBudget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("tracks LLM and tool costs separately while enforcing the total budget", () => {
|
||||||
|
const budget = new BudgetTracker({
|
||||||
|
enabled: true,
|
||||||
|
maxCostUsd: 0.0055,
|
||||||
|
maxHandoffs: 5,
|
||||||
|
maxSameAgentVisits: 5,
|
||||||
|
maxToolCalls: 5,
|
||||||
|
});
|
||||||
|
|
||||||
|
for (let index = 0; index < 5; index += 1) {
|
||||||
|
budget.recordLlmCost(0.001);
|
||||||
|
}
|
||||||
|
budget.recordToolCall(0.0005);
|
||||||
|
|
||||||
|
expect(budget.snapshot()).toMatchObject({
|
||||||
|
estimatedCostUsd: 0.0055,
|
||||||
|
totalCostUsd: 0.0055,
|
||||||
|
totalLlmCostUsd: 0.005,
|
||||||
|
totalToolCostUsd: 0.0005,
|
||||||
|
toolCalls: 1,
|
||||||
|
});
|
||||||
|
expect(budget.assertBeforeHandoff("supervisor")).toBe("cost-budget");
|
||||||
|
});
|
||||||
|
|
||||||
it("can be disabled", () => {
|
it("can be disabled", () => {
|
||||||
const budget = new BudgetTracker({
|
const budget = new BudgetTracker({
|
||||||
enabled: false,
|
enabled: false,
|
||||||
@@ -71,6 +95,7 @@ describe("BudgetTracker", () => {
|
|||||||
budget.recordVisit("supervisor");
|
budget.recordVisit("supervisor");
|
||||||
budget.recordHandoff("moderation");
|
budget.recordHandoff("moderation");
|
||||||
budget.recordToolCall(1);
|
budget.recordToolCall(1);
|
||||||
|
budget.recordLlmCost(1);
|
||||||
|
|
||||||
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
|
expect(budget.assertBeforeHandoff("supervisor")).toBe("ok");
|
||||||
});
|
});
|
||||||
|
|||||||
+23
-4
@@ -20,13 +20,17 @@ export interface BudgetSnapshot {
|
|||||||
maxHandoffs: number;
|
maxHandoffs: number;
|
||||||
maxSameAgentVisits: number;
|
maxSameAgentVisits: number;
|
||||||
maxToolCalls: number;
|
maxToolCalls: number;
|
||||||
|
totalCostUsd: number;
|
||||||
|
totalLlmCostUsd: number;
|
||||||
|
totalToolCostUsd: number;
|
||||||
toolCalls: number;
|
toolCalls: number;
|
||||||
visits: Record<string, number>;
|
visits: Record<string, number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class BudgetTracker {
|
export class BudgetTracker {
|
||||||
private estimatedCostUsd = 0;
|
|
||||||
private handoffs = 0;
|
private handoffs = 0;
|
||||||
|
private totalLlmCostUsd = 0;
|
||||||
|
private totalToolCostUsd = 0;
|
||||||
private toolCalls = 0;
|
private toolCalls = 0;
|
||||||
private readonly visits = new Map<string, number>();
|
private readonly visits = new Map<string, number>();
|
||||||
|
|
||||||
@@ -53,7 +57,7 @@ export class BudgetTracker {
|
|||||||
if (this.toolCalls >= this.limits.maxToolCalls) {
|
if (this.toolCalls >= this.limits.maxToolCalls) {
|
||||||
return "tool-budget";
|
return "tool-budget";
|
||||||
}
|
}
|
||||||
if (this.estimatedCostUsd >= this.limits.maxCostUsd) {
|
if (this.totalCostUsd() >= this.limits.maxCostUsd) {
|
||||||
return "cost-budget";
|
return "cost-budget";
|
||||||
}
|
}
|
||||||
return "ok";
|
return "ok";
|
||||||
@@ -66,7 +70,14 @@ export class BudgetTracker {
|
|||||||
|
|
||||||
recordToolCall(estimatedCostUsd = 0): void {
|
recordToolCall(estimatedCostUsd = 0): void {
|
||||||
this.toolCalls += 1;
|
this.toolCalls += 1;
|
||||||
this.estimatedCostUsd += estimatedCostUsd;
|
this.totalToolCostUsd += estimatedCostUsd;
|
||||||
|
}
|
||||||
|
|
||||||
|
recordLlmCost(costUsd: number): void {
|
||||||
|
if (!Number.isFinite(costUsd) || costUsd <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.totalLlmCostUsd += costUsd;
|
||||||
}
|
}
|
||||||
|
|
||||||
recordVisit(target: string): void {
|
recordVisit(target: string): void {
|
||||||
@@ -74,15 +85,23 @@ export class BudgetTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
snapshot(): BudgetSnapshot {
|
snapshot(): BudgetSnapshot {
|
||||||
|
const totalCostUsd = this.totalCostUsd();
|
||||||
return {
|
return {
|
||||||
estimatedCostUsd: Number(this.estimatedCostUsd.toFixed(6)),
|
estimatedCostUsd: Number(totalCostUsd.toFixed(6)),
|
||||||
handoffs: this.handoffs,
|
handoffs: this.handoffs,
|
||||||
maxCostUsd: this.limits.maxCostUsd,
|
maxCostUsd: this.limits.maxCostUsd,
|
||||||
maxHandoffs: this.limits.maxHandoffs,
|
maxHandoffs: this.limits.maxHandoffs,
|
||||||
maxSameAgentVisits: this.limits.maxSameAgentVisits,
|
maxSameAgentVisits: this.limits.maxSameAgentVisits,
|
||||||
maxToolCalls: this.limits.maxToolCalls,
|
maxToolCalls: this.limits.maxToolCalls,
|
||||||
|
totalCostUsd: Number(totalCostUsd.toFixed(6)),
|
||||||
|
totalLlmCostUsd: Number(this.totalLlmCostUsd.toFixed(6)),
|
||||||
|
totalToolCostUsd: Number(this.totalToolCostUsd.toFixed(6)),
|
||||||
toolCalls: this.toolCalls,
|
toolCalls: this.toolCalls,
|
||||||
visits: Object.fromEntries(this.visits.entries()),
|
visits: Object.fromEntries(this.visits.entries()),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private totalCostUsd(): number {
|
||||||
|
return this.totalLlmCostUsd + this.totalToolCostUsd;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -90,7 +90,7 @@ export async function dispatch(
|
|||||||
return runWithContext(
|
return runWithContext(
|
||||||
{
|
{
|
||||||
...contextBase,
|
...contextBase,
|
||||||
budget: budget.snapshot(),
|
budget,
|
||||||
handoffChain: chain,
|
handoffChain: chain,
|
||||||
handoffDepth: depth,
|
handoffDepth: depth,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -9,6 +9,11 @@ export const KNOWN_SECRET_KEYS = [
|
|||||||
"GARNET_PASSWORD",
|
"GARNET_PASSWORD",
|
||||||
"MONGODB_CONNECTION_STRING",
|
"MONGODB_CONNECTION_STRING",
|
||||||
"MONGODB_PROJECT_ID",
|
"MONGODB_PROJECT_ID",
|
||||||
|
"OPENROUTER_API_KEY",
|
||||||
|
"AGENT_LLM_API_KEY",
|
||||||
|
"OPENAI_API_KEY",
|
||||||
|
"EMBEDDING_API_KEY",
|
||||||
|
"PHOENIX_API_KEY",
|
||||||
"SLACK_ALERT_WEBHOOK_URL",
|
"SLACK_ALERT_WEBHOOK_URL",
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
import { runWithContext } from "../observability/context.js";
|
||||||
|
import { NatsClient } from "./nats.js";
|
||||||
|
import { AGENT_ALERT_SUBJECT, DISCORD_MEMBER_LOOKUP_SUBJECT } from "./nats-subjects.js";
|
||||||
|
import { checkPublishAllowed } from "./nats-skill-acl.js";
|
||||||
|
|
||||||
|
describe("NATS skill ACL", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllEnvs();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("allows declared publish subjects", () => {
|
||||||
|
expect(checkPublishAllowed("onboarding", DISCORD_MEMBER_LOOKUP_SUBJECT)).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(checkPublishAllowed("supervisor", AGENT_ALERT_SUBJECT)).toBe(true);
|
||||||
|
expect(checkPublishAllowed("onboarding", AGENT_ALERT_SUBJECT)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("enforces denied publish subjects when enabled", async () => {
|
||||||
|
vi.stubEnv("AGENT_NATS_ACL_MODE", "enforce");
|
||||||
|
const client = new NatsClient();
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runWithContext(
|
||||||
|
{
|
||||||
|
chatroomId: "room-1",
|
||||||
|
requestId: "req-1",
|
||||||
|
skill: "onboarding",
|
||||||
|
startedAt: "2026-05-18T00:00:00.000Z",
|
||||||
|
trigger: "user-message",
|
||||||
|
userId: "user-1",
|
||||||
|
},
|
||||||
|
() => client.publish(AGENT_ALERT_SUBJECT, { ok: true }),
|
||||||
|
),
|
||||||
|
).rejects.toThrow(
|
||||||
|
`NATS publish denied for skill onboarding on subject ${AGENT_ALERT_SUBJECT}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
import {
|
||||||
|
AGENT_ALERT_SUBJECT,
|
||||||
|
DISCORD_MEMBER_LOOKUP_SUBJECT,
|
||||||
|
} from "./nats-subjects.js";
|
||||||
|
|
||||||
|
export interface SkillNatsAcl {
|
||||||
|
skill: string;
|
||||||
|
allowedPublishSubjects: string[];
|
||||||
|
allowedSubscribeSubjects: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_NATS_ACL: SkillNatsAcl[] = [
|
||||||
|
{
|
||||||
|
skill: "onboarding",
|
||||||
|
allowedPublishSubjects: [DISCORD_MEMBER_LOOKUP_SUBJECT],
|
||||||
|
allowedSubscribeSubjects: [],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
skill: "moderation",
|
||||||
|
allowedPublishSubjects: [],
|
||||||
|
allowedSubscribeSubjects: [],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
skill: "supervisor",
|
||||||
|
allowedPublishSubjects: [AGENT_ALERT_SUBJECT],
|
||||||
|
allowedSubscribeSubjects: [],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
function wildcardToRegExp(pattern: string): RegExp {
|
||||||
|
const escaped = pattern
|
||||||
|
.split("*")
|
||||||
|
.map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"))
|
||||||
|
.join(".*");
|
||||||
|
return new RegExp(`^${escaped}$`, "u");
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesAny(subject: string, patterns: readonly string[]): boolean {
|
||||||
|
return patterns.some((pattern) => wildcardToRegExp(pattern).test(subject));
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAcl(skill: string): SkillNatsAcl | undefined {
|
||||||
|
return DEFAULT_NATS_ACL.find((entry) => entry.skill === skill);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function checkPublishAllowed(skill: string, subject: string): boolean {
|
||||||
|
const acl = getAcl(skill);
|
||||||
|
return Boolean(acl && matchesAny(subject, acl.allowedPublishSubjects));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function checkSubscribeAllowed(skill: string, subject: string): boolean {
|
||||||
|
const acl = getAcl(skill);
|
||||||
|
return Boolean(acl && matchesAny(subject, acl.allowedSubscribeSubjects));
|
||||||
|
}
|
||||||
@@ -6,7 +6,13 @@ import {
|
|||||||
type Subscription,
|
type Subscription,
|
||||||
} from "nats";
|
} from "nats";
|
||||||
import { getSecret } from "@common/secret-manager";
|
import { getSecret } from "@common/secret-manager";
|
||||||
|
import { getNatsAclMode } from "../config.js";
|
||||||
|
import { getContext } from "../observability/context.js";
|
||||||
import { getLogger } from "@obs/logger";
|
import { getLogger } from "@obs/logger";
|
||||||
|
import {
|
||||||
|
checkPublishAllowed,
|
||||||
|
checkSubscribeAllowed,
|
||||||
|
} from "./nats-skill-acl.js";
|
||||||
|
|
||||||
export interface NatsMessageContext<T = unknown> {
|
export interface NatsMessageContext<T = unknown> {
|
||||||
data: T;
|
data: T;
|
||||||
@@ -81,6 +87,7 @@ export class NatsClient {
|
|||||||
subject: string,
|
subject: string,
|
||||||
handler: (msg: Msg) => void | Promise<void>,
|
handler: (msg: Msg) => void | Promise<void>,
|
||||||
): Promise<Subscription> {
|
): Promise<Subscription> {
|
||||||
|
this.checkSubjectAcl("subscribe", subject);
|
||||||
const subscription = this.getConnection().subscribe(subject);
|
const subscription = this.getConnection().subscribe(subject);
|
||||||
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
@@ -101,6 +108,7 @@ export class NatsClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async publish<T = unknown>(subject: string, data: T): Promise<void> {
|
async publish<T = unknown>(subject: string, data: T): Promise<void> {
|
||||||
|
this.checkSubjectAcl("publish", subject);
|
||||||
const client = this.getConnection();
|
const client = this.getConnection();
|
||||||
client.publish(subject, this.stringCodec.encode(JSON.stringify(data)));
|
client.publish(subject, this.stringCodec.encode(JSON.stringify(data)));
|
||||||
await client.flush();
|
await client.flush();
|
||||||
@@ -111,6 +119,7 @@ export class NatsClient {
|
|||||||
data: TRequest,
|
data: TRequest,
|
||||||
options?: { timeoutMs?: number | undefined },
|
options?: { timeoutMs?: number | undefined },
|
||||||
): Promise<TResponse> {
|
): Promise<TResponse> {
|
||||||
|
this.checkSubjectAcl("publish", subject);
|
||||||
const requestOptions =
|
const requestOptions =
|
||||||
options?.timeoutMs !== undefined
|
options?.timeoutMs !== undefined
|
||||||
? {
|
? {
|
||||||
@@ -126,6 +135,36 @@ export class NatsClient {
|
|||||||
return this.decodeMessage<TResponse>(response);
|
return this.decodeMessage<TResponse>(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private checkSubjectAcl(
|
||||||
|
operation: "publish" | "subscribe",
|
||||||
|
subject: string,
|
||||||
|
): void {
|
||||||
|
const mode = getNatsAclMode();
|
||||||
|
if (mode === "off") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const skill = getContext()?.skill;
|
||||||
|
if (!skill) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const allowed =
|
||||||
|
operation === "publish"
|
||||||
|
? checkPublishAllowed(skill, subject)
|
||||||
|
: checkSubscribeAllowed(skill, subject);
|
||||||
|
if (allowed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload = { mode, operation, skill, subject };
|
||||||
|
if (mode === "enforce") {
|
||||||
|
this.logger.error(payload, "[NATS ACL] Subject denied");
|
||||||
|
throw new Error(
|
||||||
|
`NATS ${operation} denied for skill ${skill} on subject ${subject}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
this.logger.warn(payload, "[NATS ACL] Subject would be denied");
|
||||||
|
}
|
||||||
|
|
||||||
respond(msg: Msg, data: unknown): Promise<void> {
|
respond(msg: Msg, data: unknown): Promise<void> {
|
||||||
if (!msg.reply) {
|
if (!msg.reply) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
|
|||||||
@@ -5,10 +5,15 @@ import {
|
|||||||
getAlertThresholdEvalScoreDropPct,
|
getAlertThresholdEvalScoreDropPct,
|
||||||
getAlertThresholdLlmErrorRate,
|
getAlertThresholdLlmErrorRate,
|
||||||
getAlertThresholdP95LatencyMs,
|
getAlertThresholdP95LatencyMs,
|
||||||
|
getAgentBudgetLlmCostEnabled,
|
||||||
|
getAgentCostAwareTierEnabled,
|
||||||
|
getAgentSupervisorFastPathEnabled,
|
||||||
|
getAgentToolSandboxEnabled,
|
||||||
getEmbeddingModel,
|
getEmbeddingModel,
|
||||||
getEmbeddingProvider,
|
getEmbeddingProvider,
|
||||||
getLongTermSummaryEnabled,
|
getLongTermSummaryEnabled,
|
||||||
getMemoryEvictionEnabled,
|
getMemoryEvictionEnabled,
|
||||||
|
getNatsAclMode,
|
||||||
getPhoenixLogPrompts,
|
getPhoenixLogPrompts,
|
||||||
getPhoenixOtlpEndpoint,
|
getPhoenixOtlpEndpoint,
|
||||||
getRagHybridEnabled,
|
getRagHybridEnabled,
|
||||||
@@ -17,7 +22,9 @@ import {
|
|||||||
getRagEnabled,
|
getRagEnabled,
|
||||||
getSlackAlertWebhookUrl,
|
getSlackAlertWebhookUrl,
|
||||||
getSlackWebhookTimeoutMs,
|
getSlackWebhookTimeoutMs,
|
||||||
|
getSupervisorFastPathConfidence,
|
||||||
getSupervisorLlmResponseEnabled,
|
getSupervisorLlmResponseEnabled,
|
||||||
|
getToolAuditMaxLen,
|
||||||
} from "./config.js";
|
} from "./config.js";
|
||||||
|
|
||||||
const ENV_KEYS = [
|
const ENV_KEYS = [
|
||||||
@@ -35,6 +42,13 @@ const ENV_KEYS = [
|
|||||||
"AGENT_RAG_RERANKER_TOP_K",
|
"AGENT_RAG_RERANKER_TOP_K",
|
||||||
"AGENT_LONG_TERM_SUMMARY_ENABLED",
|
"AGENT_LONG_TERM_SUMMARY_ENABLED",
|
||||||
"AGENT_MEMORY_EVICTION_ENABLED",
|
"AGENT_MEMORY_EVICTION_ENABLED",
|
||||||
|
"AGENT_BUDGET_LLM_COST_ENABLED",
|
||||||
|
"AGENT_COST_AWARE_TIER_ENABLED",
|
||||||
|
"AGENT_SUPERVISOR_FAST_PATH_ENABLED",
|
||||||
|
"SUPERVISOR_FAST_PATH_CONFIDENCE",
|
||||||
|
"AGENT_TOOL_SANDBOX_ENABLED",
|
||||||
|
"AGENT_TOOL_AUDIT_MAX_LEN",
|
||||||
|
"AGENT_NATS_ACL_MODE",
|
||||||
"EMBEDDING_MODEL",
|
"EMBEDDING_MODEL",
|
||||||
"EMBEDDING_PROVIDER",
|
"EMBEDDING_PROVIDER",
|
||||||
"PHOENIX_LOG_PROMPTS",
|
"PHOENIX_LOG_PROMPTS",
|
||||||
@@ -129,4 +143,30 @@ describe("config", () => {
|
|||||||
|
|
||||||
expect(getSupervisorLlmResponseEnabled()).toBe(false);
|
expect(getSupervisorLlmResponseEnabled()).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reads operational excellence guardrail configuration", () => {
|
||||||
|
expect(getAgentBudgetLlmCostEnabled()).toBe(true);
|
||||||
|
expect(getAgentCostAwareTierEnabled()).toBe(true);
|
||||||
|
expect(getAgentSupervisorFastPathEnabled()).toBe(true);
|
||||||
|
expect(getSupervisorFastPathConfidence()).toBe(0.85);
|
||||||
|
expect(getAgentToolSandboxEnabled()).toBe(false);
|
||||||
|
expect(getToolAuditMaxLen()).toBe(100_000);
|
||||||
|
expect(getNatsAclMode()).toBe("log-only");
|
||||||
|
|
||||||
|
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
|
||||||
|
vi.stubEnv("AGENT_COST_AWARE_TIER_ENABLED", "false");
|
||||||
|
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
|
||||||
|
vi.stubEnv("SUPERVISOR_FAST_PATH_CONFIDENCE", "0.9");
|
||||||
|
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
|
||||||
|
vi.stubEnv("AGENT_TOOL_AUDIT_MAX_LEN", "42");
|
||||||
|
vi.stubEnv("AGENT_NATS_ACL_MODE", "enforce");
|
||||||
|
|
||||||
|
expect(getAgentBudgetLlmCostEnabled()).toBe(false);
|
||||||
|
expect(getAgentCostAwareTierEnabled()).toBe(false);
|
||||||
|
expect(getAgentSupervisorFastPathEnabled()).toBe(false);
|
||||||
|
expect(getSupervisorFastPathConfidence()).toBe(0.9);
|
||||||
|
expect(getAgentToolSandboxEnabled()).toBe(true);
|
||||||
|
expect(getToolAuditMaxLen()).toBe(42);
|
||||||
|
expect(getNatsAclMode()).toBe("enforce");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -231,6 +231,38 @@ export function getAgentBudgetMaxCostUsd(): number {
|
|||||||
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
|
return getPositiveNumber("AGENT_BUDGET_MAX_COST_USD", 0.5);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getAgentBudgetLlmCostEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_BUDGET_LLM_COST_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentCostAwareTierEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_COST_AWARE_TIER_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentSupervisorFastPathEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_SUPERVISOR_FAST_PATH_ENABLED", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getSupervisorFastPathConfidence(): number {
|
||||||
|
return getRatioNumber("SUPERVISOR_FAST_PATH_CONFIDENCE", 0.85);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAgentToolSandboxEnabled(): boolean {
|
||||||
|
return getBoolean("AGENT_TOOL_SANDBOX_ENABLED", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getToolAuditMaxLen(): number {
|
||||||
|
return getPositiveNumber("AGENT_TOOL_AUDIT_MAX_LEN", 100_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getNatsAclMode(): "off" | "log-only" | "enforce" {
|
||||||
|
const raw = process.env["AGENT_NATS_ACL_MODE"]?.trim().toLowerCase();
|
||||||
|
if (raw === "off" || raw === "log-only" || raw === "enforce") {
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
return "log-only";
|
||||||
|
}
|
||||||
|
|
||||||
export function getAgentRuntimeVerifierEnabled(): boolean {
|
export function getAgentRuntimeVerifierEnabled(): boolean {
|
||||||
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
|
return getBoolean("AGENT_RUNTIME_VERIFIER_ENABLED", false);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ vi.mock("@obs/spans", () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
import { LlmClient } from "./llm-client.js";
|
import { LlmClient } from "./llm-client.js";
|
||||||
|
import { BudgetTracker } from "@agent/budget";
|
||||||
|
import { runWithContext } from "@obs/context";
|
||||||
import type { ChatCompletionResponse } from "./types.js";
|
import type { ChatCompletionResponse } from "./types.js";
|
||||||
|
|
||||||
function okResponse(
|
function okResponse(
|
||||||
@@ -168,4 +170,103 @@ describe("LlmClient endpoint normalization", () => {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("records LLM cost into the request budget", async () => {
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn(() =>
|
||||||
|
Promise.resolve(
|
||||||
|
okResponse("reply", {
|
||||||
|
usage: {
|
||||||
|
completion_tokens: 50,
|
||||||
|
prompt_tokens: 100,
|
||||||
|
total_tokens: 150,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const budget = new BudgetTracker({
|
||||||
|
enabled: true,
|
||||||
|
maxCostUsd: 1,
|
||||||
|
maxHandoffs: 5,
|
||||||
|
maxSameAgentVisits: 5,
|
||||||
|
maxToolCalls: 5,
|
||||||
|
});
|
||||||
|
const client = new LlmClient({
|
||||||
|
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
|
||||||
|
providerApiKeys: { openrouter: "test-key" },
|
||||||
|
});
|
||||||
|
|
||||||
|
await runWithContext(
|
||||||
|
{
|
||||||
|
budget,
|
||||||
|
chatroomId: "room-1",
|
||||||
|
requestId: "req-1",
|
||||||
|
skill: "onboarding",
|
||||||
|
startedAt: "2026-05-18T00:00:00.000Z",
|
||||||
|
trigger: "user-message",
|
||||||
|
userId: "user-1",
|
||||||
|
},
|
||||||
|
() =>
|
||||||
|
client.chatCompletion([{ content: "hi", role: "user" }], {
|
||||||
|
modelOverride: "gpt-5.3-codex-spark",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(budget.snapshot().totalLlmCostUsd).toBe(0.000045);
|
||||||
|
expect(llmSpanMock.setAttributes).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
"budget.llm_cost_usd": 0.000045,
|
||||||
|
"budget.total_cost_usd": 0.000045,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not record LLM cost when budget cost tracking is disabled", async () => {
|
||||||
|
vi.stubEnv("AGENT_BUDGET_LLM_COST_ENABLED", "false");
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn(() =>
|
||||||
|
Promise.resolve(
|
||||||
|
okResponse("reply", {
|
||||||
|
usage: {
|
||||||
|
completion_tokens: 50,
|
||||||
|
prompt_tokens: 100,
|
||||||
|
total_tokens: 150,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const budget = new BudgetTracker({
|
||||||
|
enabled: true,
|
||||||
|
maxCostUsd: 1,
|
||||||
|
maxHandoffs: 5,
|
||||||
|
maxSameAgentVisits: 5,
|
||||||
|
maxToolCalls: 5,
|
||||||
|
});
|
||||||
|
const client = new LlmClient({
|
||||||
|
endpoints: [{ provider: "openrouter", url: "https://openrouter.ai/api/v1" }],
|
||||||
|
providerApiKeys: { openrouter: "test-key" },
|
||||||
|
});
|
||||||
|
|
||||||
|
await runWithContext(
|
||||||
|
{
|
||||||
|
budget,
|
||||||
|
chatroomId: "room-1",
|
||||||
|
requestId: "req-1",
|
||||||
|
skill: "onboarding",
|
||||||
|
startedAt: "2026-05-18T00:00:00.000Z",
|
||||||
|
trigger: "user-message",
|
||||||
|
userId: "user-1",
|
||||||
|
},
|
||||||
|
() =>
|
||||||
|
client.chatCompletion([{ content: "hi", role: "user" }], {
|
||||||
|
modelOverride: "gpt-5.3-codex-spark",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(budget.snapshot().totalLlmCostUsd).toBe(0);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -10,6 +10,7 @@ import type {
|
|||||||
import { getContext, getRequestLogger } from "@obs/context";
|
import { getContext, getRequestLogger } from "@obs/context";
|
||||||
import { maskTraceContent, withLlmSpan } from "@obs/spans";
|
import { maskTraceContent, withLlmSpan } from "@obs/spans";
|
||||||
import {
|
import {
|
||||||
|
getAgentBudgetLlmCostEnabled,
|
||||||
getLlmHealthTimeoutMs,
|
getLlmHealthTimeoutMs,
|
||||||
getLlmRequestTimeoutMs,
|
getLlmRequestTimeoutMs,
|
||||||
getLlmTemperature,
|
getLlmTemperature,
|
||||||
@@ -257,11 +258,22 @@ export class LlmClient {
|
|||||||
model: request.model,
|
model: request.model,
|
||||||
skill: getContext()?.skill,
|
skill: getContext()?.skill,
|
||||||
});
|
});
|
||||||
|
if (getAgentBudgetLlmCostEnabled()) {
|
||||||
|
getContext()?.budget?.recordLlmCost(costUsd);
|
||||||
|
}
|
||||||
|
const budgetSnapshot = getContext()?.budget?.snapshot();
|
||||||
|
|
||||||
span.setAttributes({
|
span.setAttributes({
|
||||||
...(shouldLogPrompts
|
...(shouldLogPrompts
|
||||||
? buildCompletionTraceAttributes(result)
|
? buildCompletionTraceAttributes(result)
|
||||||
: {}),
|
: {}),
|
||||||
|
...(budgetSnapshot
|
||||||
|
? {
|
||||||
|
"budget.llm_cost_usd": budgetSnapshot.totalLlmCostUsd,
|
||||||
|
"budget.tool_cost_usd": budgetSnapshot.totalToolCostUsd,
|
||||||
|
"budget.total_cost_usd": budgetSnapshot.totalCostUsd,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
"gen_ai.response.finish_reasons": payload.choices
|
"gen_ai.response.finish_reasons": payload.choices
|
||||||
?.map((choice) => choice.finish_reason ?? "unknown")
|
?.map((choice) => choice.finish_reason ?? "unknown")
|
||||||
.filter((reason): reason is string => Boolean(reason)),
|
.filter((reason): reason is string => Boolean(reason)),
|
||||||
|
|||||||
@@ -3,12 +3,12 @@ import { AsyncLocalStorage } from 'node:async_hooks';
|
|||||||
import type { Logger } from 'pino';
|
import type { Logger } from 'pino';
|
||||||
|
|
||||||
import { getLogger } from './logger.js';
|
import { getLogger } from './logger.js';
|
||||||
import type { BudgetSnapshot } from '@agent/budget';
|
import type { BudgetTracker } from '@agent/budget';
|
||||||
|
|
||||||
export interface RequestContext {
|
export interface RequestContext {
|
||||||
chatroomId: string;
|
chatroomId: string;
|
||||||
deadlineAt?: string;
|
deadlineAt?: string;
|
||||||
budget?: BudgetSnapshot | undefined;
|
budget?: BudgetTracker | undefined;
|
||||||
handoffChain?: string[] | undefined;
|
handoffChain?: string[] | undefined;
|
||||||
handoffDepth?: number | undefined;
|
handoffDepth?: number | undefined;
|
||||||
logger: Logger;
|
logger: Logger;
|
||||||
|
|||||||
@@ -15,9 +15,21 @@ type RequestBindings = {
|
|||||||
function buildRedactionPaths(): string[] {
|
function buildRedactionPaths(): string[] {
|
||||||
const commonPaths = [
|
const commonPaths = [
|
||||||
'authorization',
|
'authorization',
|
||||||
|
'apiKey',
|
||||||
|
'api_key',
|
||||||
'headers.authorization',
|
'headers.authorization',
|
||||||
|
'headers.x-api-key',
|
||||||
|
'headers["x-api-key"]',
|
||||||
'req.headers.authorization',
|
'req.headers.authorization',
|
||||||
|
'req.headers.x-api-key',
|
||||||
|
'req.headers["x-api-key"]',
|
||||||
'request.headers.authorization',
|
'request.headers.authorization',
|
||||||
|
'request.headers.x-api-key',
|
||||||
|
'request.headers["x-api-key"]',
|
||||||
|
'request.apiKey',
|
||||||
|
'request.api_key',
|
||||||
|
'response.apiKey',
|
||||||
|
'response.api_key',
|
||||||
];
|
];
|
||||||
|
|
||||||
const secretPaths = KNOWN_SECRET_KEYS.flatMap((key) => [
|
const secretPaths = KNOWN_SECRET_KEYS.flatMap((key) => [
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import {
|
|||||||
formatReviewList,
|
formatReviewList,
|
||||||
formatReviewShow,
|
formatReviewShow,
|
||||||
formatReviewStats,
|
formatReviewStats,
|
||||||
|
formatToolAuditList,
|
||||||
|
formatToolAuditShow,
|
||||||
} from "./cli.js";
|
} from "./cli.js";
|
||||||
|
|
||||||
describe("operations review CLI formatters", () => {
|
describe("operations review CLI formatters", () => {
|
||||||
@@ -81,4 +83,25 @@ describe("operations review CLI formatters", () => {
|
|||||||
}),
|
}),
|
||||||
).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2");
|
).toContain("low-confidence\ttotal=3\tpending=1\tresolved=2");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("formats tool audit rows", () => {
|
||||||
|
const entry = {
|
||||||
|
argsHash: "abc123",
|
||||||
|
calledAt: "2026-05-18T00:00:00.000Z",
|
||||||
|
chatroomId: "room-1",
|
||||||
|
durationMs: 12,
|
||||||
|
id: "1-0",
|
||||||
|
ok: true,
|
||||||
|
resultSummary: "{\"ok\":true}",
|
||||||
|
skill: "onboarding",
|
||||||
|
toolName: "validate-nickname",
|
||||||
|
type: "tool-call" as const,
|
||||||
|
userId: "user-1",
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(formatToolAuditList([entry])).toContain(
|
||||||
|
"1-0\t2026-05-18T00:00:00.000Z\tonboarding\tvalidate-nickname\tyes\t12",
|
||||||
|
);
|
||||||
|
expect(formatToolAuditShow(entry)).toContain("argsHash: abc123");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
+99
-1
@@ -15,6 +15,10 @@ import {
|
|||||||
type SupervisorReviewResolution,
|
type SupervisorReviewResolution,
|
||||||
type SupervisorReviewStats,
|
type SupervisorReviewStats,
|
||||||
} from "@skills/supervisor/review-queue";
|
} from "@skills/supervisor/review-queue";
|
||||||
|
import {
|
||||||
|
ToolAuditLogger,
|
||||||
|
type ListedToolAuditEntry,
|
||||||
|
} from "@tools/audit";
|
||||||
import {
|
import {
|
||||||
getFlagString,
|
getFlagString,
|
||||||
hasFlag,
|
hasFlag,
|
||||||
@@ -164,6 +168,46 @@ export function formatReviewStats(stats: SupervisorReviewStats): string {
|
|||||||
].join("\n");
|
].join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function formatToolAuditList(items: ListedToolAuditEntry[]): string {
|
||||||
|
const rows = [
|
||||||
|
[
|
||||||
|
"id",
|
||||||
|
"calledAt",
|
||||||
|
"skill",
|
||||||
|
"toolName",
|
||||||
|
"ok",
|
||||||
|
"durationMs",
|
||||||
|
"errorKind",
|
||||||
|
],
|
||||||
|
...items.map((item) => [
|
||||||
|
item.id,
|
||||||
|
item.calledAt,
|
||||||
|
item.skill,
|
||||||
|
item.toolName,
|
||||||
|
item.ok ? "yes" : "no",
|
||||||
|
String(item.durationMs),
|
||||||
|
item.errorKind ?? "-",
|
||||||
|
]),
|
||||||
|
];
|
||||||
|
return rows.map((row) => row.join("\t")).join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatToolAuditShow(item: ListedToolAuditEntry): string {
|
||||||
|
return [
|
||||||
|
`id: ${item.id}`,
|
||||||
|
`calledAt: ${item.calledAt}`,
|
||||||
|
`skill: ${item.skill}`,
|
||||||
|
`toolName: ${item.toolName}`,
|
||||||
|
`chatroomId: ${item.chatroomId}`,
|
||||||
|
`userId: ${item.userId}`,
|
||||||
|
`ok: ${item.ok ? "yes" : "no"}`,
|
||||||
|
`durationMs: ${item.durationMs}`,
|
||||||
|
`errorKind: ${item.errorKind ?? "-"}`,
|
||||||
|
`argsHash: ${item.argsHash}`,
|
||||||
|
`resultSummary: ${item.resultSummary || "-"}`,
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
async function handleReviewList(
|
async function handleReviewList(
|
||||||
flags: Map<string, string | true>,
|
flags: Map<string, string | true>,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
@@ -277,6 +321,52 @@ async function handleReviewStats(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function handleToolAuditList(
|
||||||
|
flags: Map<string, string | true>,
|
||||||
|
): Promise<void> {
|
||||||
|
const connection = await connectGarnetForCli();
|
||||||
|
try {
|
||||||
|
const items = await new ToolAuditLogger(connection.garnetClient).list({
|
||||||
|
limit: parsePositiveInteger(getFlagString(flags, "limit")) ?? 50,
|
||||||
|
since: sinceDurationToIso(getFlagString(flags, "since") ?? "24h"),
|
||||||
|
skill: getFlagString(flags, "skill"),
|
||||||
|
});
|
||||||
|
console.info(
|
||||||
|
hasFlag(flags, "json")
|
||||||
|
? JSON.stringify(items, null, 2)
|
||||||
|
: formatToolAuditList(items),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await connection.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleToolAuditShow(
|
||||||
|
id: string | undefined,
|
||||||
|
flags: Map<string, string | true>,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!id?.trim()) {
|
||||||
|
throw new Error("tool-audit:show requires an entry id");
|
||||||
|
}
|
||||||
|
|
||||||
|
const connection = await connectGarnetForCli();
|
||||||
|
try {
|
||||||
|
const item = await new ToolAuditLogger(connection.garnetClient).get(id);
|
||||||
|
if (!item) {
|
||||||
|
process.exitCode = 2;
|
||||||
|
console.error(`[tool-audit] entry not found: ${id}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.info(
|
||||||
|
hasFlag(flags, "json")
|
||||||
|
? JSON.stringify(item, null, 2)
|
||||||
|
: formatToolAuditShow(item),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await connection.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
async function main(): Promise<void> {
|
||||||
loadEnvironmentFromNodeEnv();
|
loadEnvironmentFromNodeEnv();
|
||||||
initLogger();
|
initLogger();
|
||||||
@@ -298,9 +388,17 @@ async function main(): Promise<void> {
|
|||||||
await handleReviewStats(flags);
|
await handleReviewStats(flags);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (command === "tool-audit:list") {
|
||||||
|
await handleToolAuditList(flags);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (command === "tool-audit:show") {
|
||||||
|
await handleToolAuditShow(positionals[0], flags);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"Unknown command. Use review:list, review:show, review:resolve, or review:stats.",
|
"Unknown command. Use review:list, review:show, review:resolve, review:stats, tool-audit:list, or tool-audit:show.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,9 @@ describe("escalation trigger", () => {
|
|||||||
maxHandoffs: 4,
|
maxHandoffs: 4,
|
||||||
maxSameAgentVisits: 2,
|
maxSameAgentVisits: 2,
|
||||||
maxToolCalls: 20,
|
maxToolCalls: 20,
|
||||||
|
totalCostUsd: 0,
|
||||||
|
totalLlmCostUsd: 0,
|
||||||
|
totalToolCostUsd: 0,
|
||||||
toolCalls: 0,
|
toolCalls: 0,
|
||||||
visits: {},
|
visits: {},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -16,5 +16,6 @@ export function createOnboardingSkill(deps: SkillDeps): Skill {
|
|||||||
deps.shadowEvalBus,
|
deps.shadowEvalBus,
|
||||||
deps.natsClient,
|
deps.natsClient,
|
||||||
deps.memoryService,
|
deps.memoryService,
|
||||||
|
deps.escalationHistoryService,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -102,9 +102,9 @@ describe("decidePreGenerationRouting", () => {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
confidence: 0.3,
|
confidence: 0.3,
|
||||||
expected: { kind: "tier0-only" },
|
expected: { kind: "tier1-direct", skipPlanner: false },
|
||||||
intentKind: "off-topic-to-other" as const,
|
intentKind: "off-topic-to-other" as const,
|
||||||
name: "keeps low-confidence off-topic on tier0",
|
name: "promotes low-confidence off-topic to tier1",
|
||||||
phase: { kind: "fresh" } as OnboardingPhase,
|
phase: { kind: "fresh" } as OnboardingPhase,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -319,6 +319,49 @@ describe("decidePreGenerationRouting", () => {
|
|||||||
).toEqual(expected);
|
).toEqual(expected);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps high-confidence low-risk requests on tier0", () => {
|
||||||
|
expect(
|
||||||
|
decidePreGenerationRouting({
|
||||||
|
classified: classified("age-question", 0.95),
|
||||||
|
derived: derived({ kind: "deadline-near", remaining: ["photo"] }, intent("age-question"), {
|
||||||
|
tier: 1,
|
||||||
|
}),
|
||||||
|
intentRiskTier: "low",
|
||||||
|
request: request(),
|
||||||
|
session: session(),
|
||||||
|
userEscalationCount: 0,
|
||||||
|
}),
|
||||||
|
).toEqual({ kind: "tier0-only" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("promotes low-confidence high-risk requests with escalation history", () => {
|
||||||
|
expect(
|
||||||
|
decidePreGenerationRouting({
|
||||||
|
classified: classified("unknown", 0.49),
|
||||||
|
derived: derived({ kind: "fresh" }, intent("unknown")),
|
||||||
|
intentRiskTier: "high",
|
||||||
|
request: request(),
|
||||||
|
session: session(),
|
||||||
|
userEscalationCount: 4,
|
||||||
|
}),
|
||||||
|
).toEqual({ kind: "tier1-direct", skipPlanner: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("prefers tier0 during latency degradation unless safety already promoted", () => {
|
||||||
|
expect(
|
||||||
|
decidePreGenerationRouting({
|
||||||
|
classified: classified("age-question", 0.8),
|
||||||
|
derived: derived({ kind: "deadline-near", remaining: ["nickname"] }, intent("age-question"), {
|
||||||
|
tier: 1,
|
||||||
|
}),
|
||||||
|
intentRiskTier: "medium",
|
||||||
|
recentLatencyP95Ms: 10_000,
|
||||||
|
request: request(),
|
||||||
|
session: session(),
|
||||||
|
}),
|
||||||
|
).toEqual({ kind: "tier0-only" });
|
||||||
|
});
|
||||||
|
|
||||||
it("trains, saves, loads, and applies a learned router model", () => {
|
it("trains, saves, loads, and applies a learned router model", () => {
|
||||||
const lowConfidenceInput = {
|
const lowConfidenceInput = {
|
||||||
classified: classified("age-question", 0.25),
|
classified: classified("age-question", 0.25),
|
||||||
|
|||||||
@@ -3,6 +3,10 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|||||||
import { dirname } from "node:path";
|
import { dirname } from "node:path";
|
||||||
|
|
||||||
import type { AgentRequest } from "@agent/types";
|
import type { AgentRequest } from "@agent/types";
|
||||||
|
import {
|
||||||
|
getAgentCostAwareTierEnabled,
|
||||||
|
getAlertThresholdP95LatencyMs,
|
||||||
|
} from "../../../config.js";
|
||||||
import type {
|
import type {
|
||||||
ClassifiedIntent,
|
ClassifiedIntent,
|
||||||
DerivedState,
|
DerivedState,
|
||||||
@@ -58,8 +62,11 @@ export interface RouterTrainingSample {
|
|||||||
export interface RouterInput {
|
export interface RouterInput {
|
||||||
classified: ClassifiedIntent;
|
classified: ClassifiedIntent;
|
||||||
derived: DerivedState;
|
derived: DerivedState;
|
||||||
|
intentRiskTier?: "low" | "medium" | "high" | undefined;
|
||||||
|
recentLatencyP95Ms?: number | undefined;
|
||||||
request: AgentRequest;
|
request: AgentRequest;
|
||||||
session: OnboardingSession | null;
|
session: OnboardingSession | null;
|
||||||
|
userEscalationCount?: number | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const ROUTER_FEATURE_NAMES = [
|
export const ROUTER_FEATURE_NAMES = [
|
||||||
@@ -294,6 +301,43 @@ function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isLatencyDegraded(input: RouterInput): boolean {
|
||||||
|
const p95 = input.recentLatencyP95Ms;
|
||||||
|
return (
|
||||||
|
typeof p95 === "number" &&
|
||||||
|
Number.isFinite(p95) &&
|
||||||
|
p95 > getAlertThresholdP95LatencyMs("onboarding")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function maybeDecideCostAwareRouting(input: RouterInput): RoutingDecision | null {
|
||||||
|
if (!getAgentCostAwareTierEnabled()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const confidence = clamp01(input.classified.confidence);
|
||||||
|
const riskTier = input.intentRiskTier ?? "medium";
|
||||||
|
const escalationCount = Math.max(0, input.userEscalationCount ?? 0);
|
||||||
|
const lowLatency = !isLatencyDegraded(input);
|
||||||
|
|
||||||
|
if (confidence < 0.5 || (riskTier === "high" && escalationCount > 3)) {
|
||||||
|
if (input.derived.responseMode === "llm-with-tools") {
|
||||||
|
return { kind: "tier1-with-tools" };
|
||||||
|
}
|
||||||
|
return { kind: "tier1-direct", skipPlanner: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!lowLatency) {
|
||||||
|
return { kind: "tier0-only" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (confidence >= 0.9 && riskTier === "low") {
|
||||||
|
return { kind: "tier0-only" };
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
|
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
|
||||||
const { classified, derived, request, session } = input;
|
const { classified, derived, request, session } = input;
|
||||||
const { confidence, intent } = classified;
|
const { confidence, intent } = classified;
|
||||||
@@ -334,6 +378,11 @@ export function decidePreGenerationRouting(input: RouterInput): RoutingDecision
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const costAwareDecision = maybeDecideCostAwareRouting(input);
|
||||||
|
if (costAwareDecision) {
|
||||||
|
return costAwareDecision;
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
phase.kind === "nickname-rejected" ||
|
phase.kind === "nickname-rejected" ||
|
||||||
intent.kind === "discord-verify-question" ||
|
intent.kind === "discord-verify-question" ||
|
||||||
|
|||||||
@@ -142,4 +142,71 @@ describe("runToolPlan", () => {
|
|||||||
toolName: "lookup-community-rule",
|
toolName: "lookup-community-rule",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("appends audit entries for successful tool calls", async () => {
|
||||||
|
const streams: Array<{
|
||||||
|
fields: Record<string, string>;
|
||||||
|
key: string;
|
||||||
|
options?: { maxLen?: number | undefined };
|
||||||
|
}> = [];
|
||||||
|
const tool: Tool<{ topic: string }, { text: string }> = {
|
||||||
|
name: "lookup-community-rule",
|
||||||
|
description: "규칙 조회",
|
||||||
|
requires: [],
|
||||||
|
argsSchema: z.object({
|
||||||
|
topic: z.string(),
|
||||||
|
}),
|
||||||
|
resultSchema: z.object({
|
||||||
|
text: z.string(),
|
||||||
|
}),
|
||||||
|
execute: () => Promise.resolve({ text: "하트 규칙" }),
|
||||||
|
};
|
||||||
|
|
||||||
|
await runToolPlan(
|
||||||
|
{
|
||||||
|
composeWith: "all-then-llm",
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
id: "rule",
|
||||||
|
tool: "lookup-community-rule",
|
||||||
|
argsLiteral: {
|
||||||
|
topic: "heart",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
[tool],
|
||||||
|
{
|
||||||
|
garnetClient: {
|
||||||
|
appendStream: (
|
||||||
|
key: string,
|
||||||
|
fields: Record<string, string>,
|
||||||
|
options?: { maxLen?: number | undefined },
|
||||||
|
) => {
|
||||||
|
streams.push({
|
||||||
|
fields,
|
||||||
|
key,
|
||||||
|
...(options ? { options } : {}),
|
||||||
|
});
|
||||||
|
return Promise.resolve("1-0");
|
||||||
|
},
|
||||||
|
} as never,
|
||||||
|
},
|
||||||
|
request(),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(streams[0]).toMatchObject({
|
||||||
|
key: "lolympus-agent:tool-calls",
|
||||||
|
fields: {
|
||||||
|
ok: "true",
|
||||||
|
skill: "onboarding",
|
||||||
|
toolName: "lookup-community-rule",
|
||||||
|
type: "tool-call",
|
||||||
|
},
|
||||||
|
options: {
|
||||||
|
maxLen: 100_000,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(streams[0]?.fields["argsHash"]).toEqual(expect.any(String));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -1,5 +1,8 @@
|
|||||||
import type { AgentRequest } from "@agent/types";
|
import type { AgentRequest } from "@agent/types";
|
||||||
|
import { getRequestLogger } from "@obs/context";
|
||||||
import { withToolSpan } from "@obs/spans";
|
import { withToolSpan } from "@obs/spans";
|
||||||
|
import { hashToolArgs, ToolAuditLogger } from "@tools/audit";
|
||||||
|
import { buildScopedContext } from "@tools/scoped-context";
|
||||||
import type {
|
import type {
|
||||||
Tool,
|
Tool,
|
||||||
ToolContext,
|
ToolContext,
|
||||||
@@ -208,6 +211,41 @@ function makePlanFailure(message: string, details?: unknown): ToolExecutionFailu
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function appendToolAudit(input: {
|
||||||
|
args: unknown;
|
||||||
|
ctx: ToolContext;
|
||||||
|
durationMs: number;
|
||||||
|
errorKind?: string | undefined;
|
||||||
|
ok: boolean;
|
||||||
|
request: AgentRequest;
|
||||||
|
resultSummary: string;
|
||||||
|
toolName: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
if (!input.ctx.garnetClient) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await new ToolAuditLogger(input.ctx.garnetClient).append({
|
||||||
|
type: "tool-call",
|
||||||
|
argsHash: hashToolArgs(input.args),
|
||||||
|
calledAt: new Date().toISOString(),
|
||||||
|
chatroomId: input.request.chatroomId,
|
||||||
|
durationMs: input.durationMs,
|
||||||
|
...(input.errorKind ? { errorKind: input.errorKind } : {}),
|
||||||
|
ok: input.ok,
|
||||||
|
resultSummary: input.resultSummary,
|
||||||
|
skill: input.request.skill,
|
||||||
|
toolName: input.toolName,
|
||||||
|
userId: input.request.userId,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
getRequestLogger().warn(
|
||||||
|
{ err: error, toolName: input.toolName },
|
||||||
|
"[ToolAudit] Failed to append tool audit entry",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function orderPlan(plan: ToolPlan): OrderedPlan {
|
function orderPlan(plan: ToolPlan): OrderedPlan {
|
||||||
const failures: ToolExecutionFailure[] = [];
|
const failures: ToolExecutionFailure[] = [];
|
||||||
const byId = new Map<string, ToolPlanStep>();
|
const byId = new Map<string, ToolPlanStep>();
|
||||||
@@ -278,19 +316,31 @@ async function runStep(input: {
|
|||||||
|
|
||||||
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
||||||
let args: Record<string, unknown> = {};
|
let args: Record<string, unknown> = {};
|
||||||
|
const startedAt = Date.now();
|
||||||
try {
|
try {
|
||||||
args = buildArgs(input);
|
args = buildArgs(input);
|
||||||
const validated = tool.argsSchema.parse(args);
|
const validated = tool.argsSchema.parse(args);
|
||||||
|
const scopedContext = buildScopedContext(tool.requires, input.ctx, tool.name);
|
||||||
const result = await withToolSpan(
|
const result = await withToolSpan(
|
||||||
tool.name,
|
tool.name,
|
||||||
{
|
{
|
||||||
"tool.name": tool.name,
|
"tool.name": tool.name,
|
||||||
"tool.plan.step_id": input.step.id,
|
"tool.plan.step_id": input.step.id,
|
||||||
},
|
},
|
||||||
() => tool.execute(validated, input.ctx),
|
() => tool.execute(validated, scopedContext),
|
||||||
);
|
);
|
||||||
const structuredFailure = getStructuredFailure(input.step, result);
|
const structuredFailure = getStructuredFailure(input.step, result);
|
||||||
if (structuredFailure) {
|
if (structuredFailure) {
|
||||||
|
await appendToolAudit({
|
||||||
|
args,
|
||||||
|
ctx: input.ctx,
|
||||||
|
durationMs: Math.max(0, Date.now() - startedAt),
|
||||||
|
errorKind: structuredFailure.kind,
|
||||||
|
ok: false,
|
||||||
|
request: input.request,
|
||||||
|
resultSummary: structuredFailure.message,
|
||||||
|
toolName: tool.name,
|
||||||
|
});
|
||||||
traceStep?.finish({
|
traceStep?.finish({
|
||||||
args,
|
args,
|
||||||
errorKind: structuredFailure.kind,
|
errorKind: structuredFailure.kind,
|
||||||
@@ -302,15 +352,35 @@ async function runStep(input: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
input.results[input.step.id] = result;
|
input.results[input.step.id] = result;
|
||||||
|
const resultSummary = summarizeToolResult(result);
|
||||||
|
await appendToolAudit({
|
||||||
|
args,
|
||||||
|
ctx: input.ctx,
|
||||||
|
durationMs: Math.max(0, Date.now() - startedAt),
|
||||||
|
ok: true,
|
||||||
|
request: input.request,
|
||||||
|
resultSummary,
|
||||||
|
toolName: tool.name,
|
||||||
|
});
|
||||||
traceStep?.finish({
|
traceStep?.finish({
|
||||||
args,
|
args,
|
||||||
name: tool.name,
|
name: tool.name,
|
||||||
ok: true,
|
ok: true,
|
||||||
resultSummary: summarizeToolResult(result),
|
resultSummary,
|
||||||
});
|
});
|
||||||
return null;
|
return null;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const failure = classifyError(tool.name, error);
|
const failure = classifyError(tool.name, error);
|
||||||
|
await appendToolAudit({
|
||||||
|
args,
|
||||||
|
ctx: input.ctx,
|
||||||
|
durationMs: Math.max(0, Date.now() - startedAt),
|
||||||
|
errorKind: failure.kind,
|
||||||
|
ok: false,
|
||||||
|
request: input.request,
|
||||||
|
resultSummary: failure.message,
|
||||||
|
toolName: tool.name,
|
||||||
|
});
|
||||||
traceStep?.finish({
|
traceStep?.finish({
|
||||||
args,
|
args,
|
||||||
errorKind: failure.kind,
|
errorKind: failure.kind,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
|
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
|
||||||
import type { GarnetClient } from '@common/garnet';
|
import type { GarnetClient } from '@common/garnet';
|
||||||
import type { NatsClient } from '@common/nats';
|
import type { NatsClient } from '@common/nats';
|
||||||
|
import type { EscalationHistoryService } from '../../memory/escalation-history.js';
|
||||||
import type { MemoryService } from '../../memory/memory-types.js';
|
import type { MemoryService } from '../../memory/memory-types.js';
|
||||||
import {
|
import {
|
||||||
buildLongTermSummary,
|
buildLongTermSummary,
|
||||||
@@ -118,6 +119,7 @@ export class OnboardingSkill implements Skill {
|
|||||||
shadowEvalPublisher?: ShadowEvalPublisher,
|
shadowEvalPublisher?: ShadowEvalPublisher,
|
||||||
private readonly natsClient?: Pick<NatsClient, 'request'>,
|
private readonly natsClient?: Pick<NatsClient, 'request'>,
|
||||||
private readonly memoryService?: MemoryService,
|
private readonly memoryService?: MemoryService,
|
||||||
|
private readonly escalationHistoryService?: EscalationHistoryService,
|
||||||
) {
|
) {
|
||||||
this.classifier = new OnboardingIntentClassifier(llmClient);
|
this.classifier = new OnboardingIntentClassifier(llmClient);
|
||||||
this.blackboardStore = new OnboardingBlackboardStore(
|
this.blackboardStore = new OnboardingBlackboardStore(
|
||||||
@@ -186,13 +188,17 @@ export class OnboardingSkill implements Skill {
|
|||||||
|
|
||||||
const deriveStep = trace?.startStep('derive');
|
const deriveStep = trace?.startStep('derive');
|
||||||
const derived = deriveState(session, request.trigger, classified.intent);
|
const derived = deriveState(session, request.trigger, classified.intent);
|
||||||
|
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
||||||
const routerMode = getAgentRouterMode();
|
const routerMode = getAgentRouterMode();
|
||||||
if (routerMode === 'static' || routerMode === 'learned-v1') {
|
if (routerMode === 'static' || routerMode === 'learned-v1') {
|
||||||
const routerInput = {
|
const routerInput = {
|
||||||
classified,
|
classified,
|
||||||
derived,
|
derived,
|
||||||
|
intentRiskTier: this.capabilityCard().riskTier,
|
||||||
|
recentLatencyP95Ms: this.getRecentLatencyP95Ms(request),
|
||||||
request,
|
request,
|
||||||
session,
|
session,
|
||||||
|
userEscalationCount: escalationHistory?.totalEscalations ?? 0,
|
||||||
};
|
};
|
||||||
derived.routingDecision =
|
derived.routingDecision =
|
||||||
routerMode === 'learned-v1'
|
routerMode === 'learned-v1'
|
||||||
@@ -353,6 +359,26 @@ export class OnboardingSkill implements Skill {
|
|||||||
return normalizeOnboardingSession(rawSession);
|
return normalizeOnboardingSession(rawSession);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async loadEscalationHistory(userId: string) {
|
||||||
|
if (!this.escalationHistoryService) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await this.escalationHistoryService.load(userId);
|
||||||
|
} catch (error) {
|
||||||
|
getRequestLogger().warn(
|
||||||
|
{ err: error },
|
||||||
|
'[OnboardingSkill] Failed to load escalation history',
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private getRecentLatencyP95Ms(request: AgentRequest): number | undefined {
|
||||||
|
const raw = request.context?.['recentLatencyP95Ms'];
|
||||||
|
return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
private async decideEarlyAbstention(
|
private async decideEarlyAbstention(
|
||||||
request: AgentRequest,
|
request: AgentRequest,
|
||||||
classified: ClassifiedIntent,
|
classified: ClassifiedIntent,
|
||||||
|
|||||||
@@ -1,17 +1,53 @@
|
|||||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||||
|
|
||||||
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
|
export interface SupervisorRuleRoute {
|
||||||
const text = [
|
confidence: number;
|
||||||
|
target: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RULES = [
|
||||||
|
{
|
||||||
|
pattern: /(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu,
|
||||||
|
target: "moderation",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
pattern: /(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu,
|
||||||
|
target: "onboarding",
|
||||||
|
},
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
function packetText(packet: HandoffPacket | undefined): string {
|
||||||
|
return [
|
||||||
packet?.userGoal,
|
packet?.userGoal,
|
||||||
packet?.summary,
|
packet?.summary,
|
||||||
packet?.handoffReason,
|
packet?.handoffReason,
|
||||||
].join(" ");
|
].join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
if (/(규칙|신고|제재|경고|욕설|광고|홍보|appeal|policy|moderation)/iu.test(text)) {
|
export function routeSupervisorByRules(
|
||||||
return "moderation";
|
packet: HandoffPacket | undefined,
|
||||||
|
): SupervisorRuleRoute {
|
||||||
|
const text = packetText(packet);
|
||||||
|
const matches = RULES.filter((rule) => rule.pattern.test(text));
|
||||||
|
|
||||||
|
if (matches.length === 1) {
|
||||||
|
return {
|
||||||
|
confidence: 0.9,
|
||||||
|
target: matches[0]?.target ?? null,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
if (/(가입|닉네임|하트|인증|디스코드|온보딩|onboarding)/iu.test(text)) {
|
if (matches.length > 1) {
|
||||||
return "onboarding";
|
return {
|
||||||
|
confidence: 0.6,
|
||||||
|
target: matches[0]?.target ?? null,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
return null;
|
return {
|
||||||
|
confidence: 0,
|
||||||
|
target: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function routeSupervisorTarget(packet: HandoffPacket | undefined): string | null {
|
||||||
|
return routeSupervisorByRules(packet).target;
|
||||||
}
|
}
|
||||||
@@ -119,7 +119,7 @@ describe("SupervisorSkill", () => {
|
|||||||
|
|
||||||
expect(response.text).toContain("운영자");
|
expect(response.text).toContain("운영자");
|
||||||
expect(response.metadata).toMatchObject({
|
expect(response.metadata).toMatchObject({
|
||||||
routedBy: "rules",
|
routedBy: "regex-fast-path",
|
||||||
source: "template",
|
source: "template",
|
||||||
supervisorReason: "max-hop",
|
supervisorReason: "max-hop",
|
||||||
suggestedTarget: "onboarding",
|
suggestedTarget: "onboarding",
|
||||||
@@ -149,6 +149,7 @@ describe("SupervisorSkill", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("uses LLM routing when the decision is confident and valid", async () => {
|
it("uses LLM routing when the decision is confident and valid", async () => {
|
||||||
|
vi.stubEnv("AGENT_SUPERVISOR_FAST_PATH_ENABLED", "false");
|
||||||
const garnet = new FakeGarnet();
|
const garnet = new FakeGarnet();
|
||||||
const nats = new FakeNats();
|
const nats = new FakeNats();
|
||||||
const skill = new SupervisorSkill(
|
const skill = new SupervisorSkill(
|
||||||
@@ -189,6 +190,45 @@ describe("SupervisorSkill", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses regex fast-path before calling the supervisor LLM", async () => {
|
||||||
|
const garnet = new FakeGarnet();
|
||||||
|
const nats = new FakeNats();
|
||||||
|
const llm = {
|
||||||
|
chatCompletion: vi.fn(),
|
||||||
|
};
|
||||||
|
const skill = new SupervisorSkill(
|
||||||
|
garnet,
|
||||||
|
nats,
|
||||||
|
llm as unknown as Pick<LlmClient, "chatCompletion">,
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
const packet = buildHandoffPacket({
|
||||||
|
handoff: {
|
||||||
|
confidence: 0.7,
|
||||||
|
evidenceRefs: [],
|
||||||
|
expectedOutput: "text",
|
||||||
|
reason: "out-of-scope",
|
||||||
|
risk: "medium",
|
||||||
|
targetAgent: "supervisor",
|
||||||
|
taskSpec: "광고 규칙 문의",
|
||||||
|
},
|
||||||
|
request: baseRequest,
|
||||||
|
response: sourceResponse,
|
||||||
|
sourceChain: ["onboarding", "supervisor"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await skill.handle(
|
||||||
|
{ ...baseRequest, context: { handoffPacket: packet } },
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(llm.chatCompletion).not.toHaveBeenCalled();
|
||||||
|
expect(response.metadata).toMatchObject({
|
||||||
|
routedBy: "regex-fast-path",
|
||||||
|
suggestedTarget: "moderation",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it("uses safe LLM userMessage for supervisor replies", async () => {
|
it("uses safe LLM userMessage for supervisor replies", async () => {
|
||||||
const garnet = new FakeGarnet();
|
const garnet = new FakeGarnet();
|
||||||
const nats = new FakeNats();
|
const nats = new FakeNats();
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import type {
|
|||||||
AgentResponse,
|
AgentResponse,
|
||||||
ConversationMessage,
|
ConversationMessage,
|
||||||
} from "@agent/types";
|
} from "@agent/types";
|
||||||
|
import { trace } from "@opentelemetry/api";
|
||||||
import { readHandoffPacket } from "@agent/handoff-packet";
|
import { readHandoffPacket } from "@agent/handoff-packet";
|
||||||
import type { HandoffPacket } from "@agent/handoff-packet";
|
import type { HandoffPacket } from "@agent/handoff-packet";
|
||||||
import type { GarnetClient } from "@common/garnet";
|
import type { GarnetClient } from "@common/garnet";
|
||||||
@@ -19,6 +20,8 @@ import type { Skill } from "@skills/skill";
|
|||||||
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
||||||
import { getLogger } from "@obs/logger";
|
import { getLogger } from "@obs/logger";
|
||||||
import {
|
import {
|
||||||
|
getAgentSupervisorFastPathEnabled,
|
||||||
|
getSupervisorFastPathConfidence,
|
||||||
getSupervisorLlmConfidenceThreshold,
|
getSupervisorLlmConfidenceThreshold,
|
||||||
getSupervisorLlmEnabled,
|
getSupervisorLlmEnabled,
|
||||||
getSupervisorLlmResponseEnabled,
|
getSupervisorLlmResponseEnabled,
|
||||||
@@ -28,7 +31,7 @@ import {
|
|||||||
buildSupervisorUserPrompt,
|
buildSupervisorUserPrompt,
|
||||||
} from "./capability-prompt.js";
|
} from "./capability-prompt.js";
|
||||||
import { appendSupervisorReview } from "./review-queue.js";
|
import { appendSupervisorReview } from "./review-queue.js";
|
||||||
import { routeSupervisorTarget } from "./routing-rules.js";
|
import { routeSupervisorByRules } from "./routing-rules.js";
|
||||||
import {
|
import {
|
||||||
supervisorRoutingSchema,
|
supervisorRoutingSchema,
|
||||||
type SupervisorRoutingDecision,
|
type SupervisorRoutingDecision,
|
||||||
@@ -114,13 +117,30 @@ export class SupervisorSkill implements Skill {
|
|||||||
): Promise<AgentResponse> {
|
): Promise<AgentResponse> {
|
||||||
const packet = readHandoffPacket(request);
|
const packet = readHandoffPacket(request);
|
||||||
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
||||||
const llmDecision = await this.routeByLlm(packet, escalationHistory);
|
const rulesDecision = routeSupervisorByRules(packet);
|
||||||
|
const useRegexFastPath =
|
||||||
|
getAgentSupervisorFastPathEnabled() &&
|
||||||
|
rulesDecision.target !== null &&
|
||||||
|
rulesDecision.confidence >= getSupervisorFastPathConfidence() &&
|
||||||
|
this.isKnownTarget(rulesDecision.target);
|
||||||
|
const llmDecision = useRegexFastPath
|
||||||
|
? null
|
||||||
|
: await this.routeByLlm(packet, escalationHistory);
|
||||||
const useLlmRoute =
|
const useLlmRoute =
|
||||||
llmDecision !== null &&
|
llmDecision !== null &&
|
||||||
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
llmDecision.confidence >= getSupervisorLlmConfidenceThreshold();
|
||||||
const target = useLlmRoute
|
const target = useLlmRoute
|
||||||
? llmDecision.target
|
? llmDecision.target
|
||||||
: routeSupervisorTarget(packet);
|
: rulesDecision.target;
|
||||||
|
const routedBy = useRegexFastPath
|
||||||
|
? "regex-fast-path"
|
||||||
|
: useLlmRoute
|
||||||
|
? "llm"
|
||||||
|
: "rules-fallback";
|
||||||
|
trace.getActiveSpan()?.setAttributes({
|
||||||
|
"supervisor.routed_by": routedBy,
|
||||||
|
"supervisor.rules_confidence": rulesDecision.confidence,
|
||||||
|
});
|
||||||
const reason = packet?.handoffReason ?? "out-of-scope";
|
const reason = packet?.handoffReason ?? "out-of-scope";
|
||||||
|
|
||||||
const queuedAt = new Date().toISOString();
|
const queuedAt = new Date().toISOString();
|
||||||
@@ -169,7 +189,8 @@ export class SupervisorSkill implements Skill {
|
|||||||
return {
|
return {
|
||||||
confidence: 0.6,
|
confidence: 0.6,
|
||||||
metadata: {
|
metadata: {
|
||||||
routedBy: useLlmRoute ? "llm" : "rules",
|
routedBy,
|
||||||
|
rulesConfidence: rulesDecision.confidence,
|
||||||
source: userMessage.source,
|
source: userMessage.source,
|
||||||
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
supervisorReason: useLlmRoute ? llmDecision.reason : reason,
|
||||||
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
|
supervisorEscalationTotal: escalationHistory?.totalEscalations ?? 0,
|
||||||
@@ -234,7 +255,10 @@ export class SupervisorSkill implements Skill {
|
|||||||
|
|
||||||
private isKnownTarget(target: string): boolean {
|
private isKnownTarget(target: string): boolean {
|
||||||
return (
|
return (
|
||||||
target === "human-review" || Boolean(this.skillRegistry?.get(target))
|
target === "human-review" ||
|
||||||
|
target === "onboarding" ||
|
||||||
|
target === "moderation" ||
|
||||||
|
Boolean(this.skillRegistry?.get(target))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
|
||||||
|
import type { GarnetClient } from "@common/garnet";
|
||||||
|
import { getToolAuditMaxLen } from "../config.js";
|
||||||
|
|
||||||
|
export const TOOL_AUDIT_STREAM = "lolympus-agent:tool-calls";
|
||||||
|
|
||||||
|
export interface ToolAuditEntry {
|
||||||
|
type: "tool-call";
|
||||||
|
toolName: string;
|
||||||
|
chatroomId: string;
|
||||||
|
userId: string;
|
||||||
|
argsHash: string;
|
||||||
|
resultSummary: string;
|
||||||
|
ok: boolean;
|
||||||
|
errorKind?: string | undefined;
|
||||||
|
durationMs: number;
|
||||||
|
skill: string;
|
||||||
|
traceId?: string | undefined;
|
||||||
|
calledAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ListedToolAuditEntry extends ToolAuditEntry {
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeForHash(value: unknown): unknown {
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
return value.map(normalizeForHash);
|
||||||
|
}
|
||||||
|
if (typeof value !== "object" || value === null) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value as Record<string, unknown>)
|
||||||
|
.filter(([key]) => !/(password|secret|token|api[_-]?key|authorization)/iu.test(key))
|
||||||
|
.sort(([left], [right]) => left.localeCompare(right))
|
||||||
|
.map(([key, entry]) => [key, normalizeForHash(entry)]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function asString(value: string | undefined): string | undefined {
|
||||||
|
return value?.trim() ? value : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseAuditEntry(
|
||||||
|
id: string,
|
||||||
|
fields: Record<string, string>,
|
||||||
|
): ListedToolAuditEntry | null {
|
||||||
|
if (fields["type"] !== "tool-call") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
type: "tool-call",
|
||||||
|
argsHash: fields["argsHash"] ?? "",
|
||||||
|
calledAt: fields["calledAt"] ?? "",
|
||||||
|
chatroomId: fields["chatroomId"] ?? "",
|
||||||
|
durationMs: Number(fields["durationMs"] ?? 0),
|
||||||
|
...(asString(fields["errorKind"]) ? { errorKind: fields["errorKind"] } : {}),
|
||||||
|
ok: fields["ok"] === "true",
|
||||||
|
resultSummary: fields["resultSummary"] ?? "",
|
||||||
|
skill: fields["skill"] ?? "",
|
||||||
|
...(asString(fields["traceId"]) ? { traceId: fields["traceId"] } : {}),
|
||||||
|
toolName: fields["toolName"] ?? "",
|
||||||
|
userId: fields["userId"] ?? "",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hashToolArgs(args: unknown): string {
|
||||||
|
return createHash("sha256")
|
||||||
|
.update(JSON.stringify(normalizeForHash(args)))
|
||||||
|
.digest("hex")
|
||||||
|
.slice(0, 16);
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ToolAuditLogger {
|
||||||
|
constructor(
|
||||||
|
private readonly garnetClient: Pick<
|
||||||
|
GarnetClient,
|
||||||
|
"appendStream" | "readStreamRange"
|
||||||
|
>,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async append(entry: ToolAuditEntry): Promise<void> {
|
||||||
|
await this.garnetClient.appendStream(
|
||||||
|
TOOL_AUDIT_STREAM,
|
||||||
|
Object.fromEntries(
|
||||||
|
Object.entries(entry).map(([key, value]) => [key, String(value)]),
|
||||||
|
),
|
||||||
|
{ maxLen: getToolAuditMaxLen() },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async list(opts: {
|
||||||
|
skill?: string | undefined;
|
||||||
|
since?: string | undefined;
|
||||||
|
limit?: number | undefined;
|
||||||
|
}): Promise<ListedToolAuditEntry[]> {
|
||||||
|
const limit = Math.max(1, Math.min(opts.limit ?? 50, 500));
|
||||||
|
const entries = await this.garnetClient.readStreamRange(
|
||||||
|
TOOL_AUDIT_STREAM,
|
||||||
|
"-",
|
||||||
|
"+",
|
||||||
|
{ count: limit * 4 },
|
||||||
|
);
|
||||||
|
const sinceTime = opts.since ? Date.parse(opts.since) : NaN;
|
||||||
|
|
||||||
|
return entries
|
||||||
|
.map((entry) => parseAuditEntry(entry.id, entry.message))
|
||||||
|
.filter((entry): entry is ListedToolAuditEntry => entry !== null)
|
||||||
|
.filter((entry) => !opts.skill || entry.skill === opts.skill)
|
||||||
|
.filter((entry) => {
|
||||||
|
if (!Number.isFinite(sinceTime)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const calledAtTime = Date.parse(entry.calledAt);
|
||||||
|
return Number.isFinite(calledAtTime) && calledAtTime >= sinceTime;
|
||||||
|
})
|
||||||
|
.slice(-limit)
|
||||||
|
.reverse();
|
||||||
|
}
|
||||||
|
|
||||||
|
async get(id: string): Promise<ListedToolAuditEntry | null> {
|
||||||
|
const entries = await this.garnetClient.readStreamRange(
|
||||||
|
TOOL_AUDIT_STREAM,
|
||||||
|
id,
|
||||||
|
id,
|
||||||
|
{ count: 1 },
|
||||||
|
);
|
||||||
|
const first = entries[0];
|
||||||
|
return first ? parseAuditEntry(first.id, first.message) : null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -93,6 +93,7 @@ export const lookupRuleTool: Tool<
|
|||||||
name: "lookup-rule",
|
name: "lookup-rule",
|
||||||
description:
|
description:
|
||||||
"광고, 도배, 욕설, 신고, 제재, 이의 제기, 개인정보 같은 moderation 규칙을 조회한다.",
|
"광고, 도배, 욕설, 신고, 제재, 이의 제기, 개인정보 같은 moderation 규칙을 조회한다.",
|
||||||
|
requires: ["memory-search"],
|
||||||
argsSchema,
|
argsSchema,
|
||||||
resultSchema,
|
resultSchema,
|
||||||
async execute(args, ctx) {
|
async execute(args, ctx) {
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ export const checkOnboardingProgressTool: Tool<
|
|||||||
name: "check-onboarding-progress",
|
name: "check-onboarding-progress",
|
||||||
description:
|
description:
|
||||||
"현재 유저의 온보딩 진행 상태를 읽어 카카오톡 단계와 디스코드 단계를 포함한 남은 절차를 구조화해서 반환한다.",
|
"현재 유저의 온보딩 진행 상태를 읽어 카카오톡 단계와 디스코드 단계를 포함한 남은 절차를 구조화해서 반환한다.",
|
||||||
|
requires: ["garnet", "user-session"],
|
||||||
argsSchema,
|
argsSchema,
|
||||||
resultSchema,
|
resultSchema,
|
||||||
async execute(args, ctx) {
|
async execute(args, ctx) {
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ export const getRoomContextTool: Tool<
|
|||||||
name: "get-room-context",
|
name: "get-room-context",
|
||||||
description:
|
description:
|
||||||
"최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.",
|
"최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.",
|
||||||
|
requires: ["room-context"],
|
||||||
argsSchema,
|
argsSchema,
|
||||||
resultSchema,
|
resultSchema,
|
||||||
async execute(args, ctx) {
|
async execute(args, ctx) {
|
||||||
|
|||||||
@@ -90,6 +90,7 @@ export const lookupCommunityRuleTool: Tool<
|
|||||||
name: "lookup-community-rule",
|
name: "lookup-community-rule",
|
||||||
description:
|
description:
|
||||||
"나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 조회한다.",
|
"나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 조회한다.",
|
||||||
|
requires: ["memory-search"],
|
||||||
argsSchema,
|
argsSchema,
|
||||||
resultSchema,
|
resultSchema,
|
||||||
async execute(args, ctx) {
|
async execute(args, ctx) {
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ export const validateNicknameTool: Tool<
|
|||||||
name: "validate-nickname",
|
name: "validate-nickname",
|
||||||
description:
|
description:
|
||||||
"새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.",
|
"새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.",
|
||||||
|
requires: [],
|
||||||
argsSchema,
|
argsSchema,
|
||||||
resultSchema,
|
resultSchema,
|
||||||
execute(args) {
|
execute(args) {
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ export const verifyDiscordNicknameTool: Tool<
|
|||||||
name: 'verify-discord-nickname',
|
name: 'verify-discord-nickname',
|
||||||
description:
|
description:
|
||||||
'현재 유저의 카카오톡 인증 닉네임과 세션 입장 시각을 기준으로 디스코드 멤버를 찾아, 닉네임 변경이 확인됐는지 구조화해서 반환한다.',
|
'현재 유저의 카카오톡 인증 닉네임과 세션 입장 시각을 기준으로 디스코드 멤버를 찾아, 닉네임 변경이 확인됐는지 구조화해서 반환한다.',
|
||||||
|
requires: ['garnet', 'nats', 'user-session'],
|
||||||
argsSchema,
|
argsSchema,
|
||||||
resultSchema,
|
resultSchema,
|
||||||
async execute(_args, ctx) {
|
async execute(_args, ctx) {
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
import { buildScopedContext } from "./scoped-context.js";
|
||||||
|
import type { ToolContext } from "./tool.js";
|
||||||
|
|
||||||
|
describe("buildScopedContext", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllEnvs();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns the full context when sandbox is disabled", () => {
|
||||||
|
const ctx: ToolContext = {
|
||||||
|
chatroomId: "room-1",
|
||||||
|
garnetClient: {} as ToolContext["garnetClient"],
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(buildScopedContext([], ctx, "pure-tool")).toBe(ctx);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws when sandboxed tools access undeclared permissions", () => {
|
||||||
|
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
|
||||||
|
const ctx: ToolContext = {
|
||||||
|
chatroomId: "room-1",
|
||||||
|
garnetClient: {} as ToolContext["garnetClient"],
|
||||||
|
userId: "user-1",
|
||||||
|
};
|
||||||
|
|
||||||
|
const scoped = buildScopedContext([], ctx, "validate-nickname");
|
||||||
|
|
||||||
|
expect(scoped.chatroomId).toBe("room-1");
|
||||||
|
expect(() => scoped.garnetClient).toThrow(
|
||||||
|
"Tool validate-nickname does not declare requires:garnet",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("exposes declared memory search without profile access", async () => {
|
||||||
|
vi.stubEnv("AGENT_TOOL_SANDBOX_ENABLED", "true");
|
||||||
|
const search = vi.fn(() => Promise.resolve([]));
|
||||||
|
const ctx = {
|
||||||
|
memoryService: {
|
||||||
|
longTerm: {
|
||||||
|
search,
|
||||||
|
loadUserProfile: vi.fn(),
|
||||||
|
},
|
||||||
|
session: {},
|
||||||
|
assembleWorkingMemory: vi.fn(),
|
||||||
|
},
|
||||||
|
} as unknown as ToolContext;
|
||||||
|
|
||||||
|
const scoped = buildScopedContext(
|
||||||
|
["memory-search"],
|
||||||
|
ctx,
|
||||||
|
"lookup-community-rule",
|
||||||
|
);
|
||||||
|
|
||||||
|
await scoped.memoryService?.longTerm.search("heart", { topK: 1 });
|
||||||
|
expect(search).toHaveBeenCalledWith("heart", { topK: 1 });
|
||||||
|
expect(() => scoped.memoryService?.longTerm.loadUserProfile("user-1")).toThrow(
|
||||||
|
"Tool lookup-community-rule does not declare requires:memory-profile",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
import { getAgentToolSandboxEnabled } from "../config.js";
|
||||||
|
import type { LongTermMemory, MemoryService } from "../memory/memory-types.js";
|
||||||
|
import { getLogger } from "../observability/logger.js";
|
||||||
|
import type { ToolContext, ToolPermission } from "./tool.js";
|
||||||
|
|
||||||
|
const PERMISSION_BY_FIELD: Record<string, ToolPermission> = {
|
||||||
|
garnetClient: "garnet",
|
||||||
|
memoryService: "memory-search",
|
||||||
|
natsClient: "nats",
|
||||||
|
roomContextService: "room-context",
|
||||||
|
session: "user-session",
|
||||||
|
toolRegistry: "tool-registry",
|
||||||
|
};
|
||||||
|
|
||||||
|
function deniedAccess(toolName: string, permission: ToolPermission): never {
|
||||||
|
throw new Error(`Tool ${toolName} does not declare requires:${permission}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function defineDeniedGetter<T extends object>(
|
||||||
|
target: T,
|
||||||
|
key: string,
|
||||||
|
toolName: string,
|
||||||
|
permission: ToolPermission,
|
||||||
|
): void {
|
||||||
|
Object.defineProperty(target, key, {
|
||||||
|
enumerable: true,
|
||||||
|
get: () => deniedAccess(toolName, permission),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildScopedMemoryService(
|
||||||
|
requires: ReadonlySet<ToolPermission>,
|
||||||
|
fullContext: ToolContext,
|
||||||
|
toolName: string,
|
||||||
|
): MemoryService | undefined {
|
||||||
|
if (!fullContext.memoryService) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const longTerm: Partial<LongTermMemory> = {};
|
||||||
|
if (requires.has("memory-search")) {
|
||||||
|
longTerm.search = fullContext.memoryService.longTerm.search.bind(
|
||||||
|
fullContext.memoryService.longTerm,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
defineDeniedGetter(longTerm, "search", toolName, "memory-search");
|
||||||
|
}
|
||||||
|
if (requires.has("memory-profile")) {
|
||||||
|
longTerm.loadUserProfile =
|
||||||
|
fullContext.memoryService.longTerm.loadUserProfile.bind(
|
||||||
|
fullContext.memoryService.longTerm,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
defineDeniedGetter(longTerm, "loadUserProfile", toolName, "memory-profile");
|
||||||
|
}
|
||||||
|
|
||||||
|
defineDeniedGetter(longTerm, "updateUserProfile", toolName, "memory-profile");
|
||||||
|
defineDeniedGetter(longTerm, "upsertDocument", toolName, "memory-search");
|
||||||
|
defineDeniedGetter(longTerm, "deleteDocument", toolName, "memory-search");
|
||||||
|
|
||||||
|
return {
|
||||||
|
longTerm: longTerm as LongTermMemory,
|
||||||
|
session: fullContext.memoryService.session,
|
||||||
|
assembleWorkingMemory: fullContext.memoryService.assembleWorkingMemory.bind(
|
||||||
|
fullContext.memoryService,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function findUndeclaredContextPermissions(
|
||||||
|
requires: ReadonlySet<ToolPermission>,
|
||||||
|
fullContext: ToolContext,
|
||||||
|
): ToolPermission[] {
|
||||||
|
return [
|
||||||
|
...new Set(
|
||||||
|
Object.entries(PERMISSION_BY_FIELD)
|
||||||
|
.filter(
|
||||||
|
([field, permission]) =>
|
||||||
|
!requires.has(permission) &&
|
||||||
|
fullContext[field as keyof ToolContext] !== undefined,
|
||||||
|
)
|
||||||
|
.map(([, permission]) => permission),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildScopedContext(
|
||||||
|
requires: readonly ToolPermission[] | undefined,
|
||||||
|
fullContext: ToolContext,
|
||||||
|
toolName = "unknown",
|
||||||
|
): ToolContext {
|
||||||
|
const sandboxEnabled = getAgentToolSandboxEnabled();
|
||||||
|
if (!requires) {
|
||||||
|
if (sandboxEnabled) {
|
||||||
|
getLogger().warn(
|
||||||
|
{ toolName },
|
||||||
|
"[ToolSandbox] Tool has no requires declaration; using full context",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return fullContext;
|
||||||
|
}
|
||||||
|
if (!sandboxEnabled) {
|
||||||
|
const undeclared = findUndeclaredContextPermissions(
|
||||||
|
new Set(requires),
|
||||||
|
fullContext,
|
||||||
|
);
|
||||||
|
if (undeclared.length > 0) {
|
||||||
|
getLogger().warn(
|
||||||
|
{ toolName, undeclared },
|
||||||
|
"[ToolSandbox] Tool context includes undeclared permissions",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return fullContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
const required = new Set(requires);
|
||||||
|
const scoped: ToolContext = {
|
||||||
|
chatroomId: fullContext.chatroomId,
|
||||||
|
userId: fullContext.userId,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (required.has("garnet")) {
|
||||||
|
scoped.garnetClient = fullContext.garnetClient;
|
||||||
|
}
|
||||||
|
if (required.has("nats")) {
|
||||||
|
scoped.natsClient = fullContext.natsClient;
|
||||||
|
}
|
||||||
|
if (required.has("room-context")) {
|
||||||
|
scoped.roomContextService = fullContext.roomContextService;
|
||||||
|
}
|
||||||
|
if (required.has("tool-registry")) {
|
||||||
|
scoped.toolRegistry = fullContext.toolRegistry;
|
||||||
|
}
|
||||||
|
if (required.has("user-session")) {
|
||||||
|
scoped.session = fullContext.session;
|
||||||
|
}
|
||||||
|
if (required.has("memory-search") || required.has("memory-profile")) {
|
||||||
|
scoped.memoryService = buildScopedMemoryService(required, fullContext, toolName);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [field, permission] of Object.entries(PERMISSION_BY_FIELD)) {
|
||||||
|
if (field in scoped) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
defineDeniedGetter(scoped, field, toolName, permission);
|
||||||
|
}
|
||||||
|
|
||||||
|
return scoped;
|
||||||
|
}
|
||||||
@@ -17,6 +17,15 @@ export interface ToolContext {
|
|||||||
toolRegistry?: ToolRegistry | undefined;
|
toolRegistry?: ToolRegistry | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type ToolPermission =
|
||||||
|
| "garnet"
|
||||||
|
| "memory-profile"
|
||||||
|
| "memory-search"
|
||||||
|
| "nats"
|
||||||
|
| "room-context"
|
||||||
|
| "tool-registry"
|
||||||
|
| "user-session";
|
||||||
|
|
||||||
export type ToolErrorKind =
|
export type ToolErrorKind =
|
||||||
| "args-invalid"
|
| "args-invalid"
|
||||||
| "transient"
|
| "transient"
|
||||||
@@ -34,6 +43,7 @@ export interface ToolExecutionFailure {
|
|||||||
export interface Tool<A, R> {
|
export interface Tool<A, R> {
|
||||||
name: string;
|
name: string;
|
||||||
description: string;
|
description: string;
|
||||||
|
requires?: readonly ToolPermission[] | undefined;
|
||||||
argsSchema: ZodType<A>;
|
argsSchema: ZodType<A>;
|
||||||
resultSchema: ZodType<R>;
|
resultSchema: ZodType<R>;
|
||||||
execute(args: A, ctx: ToolContext): Promise<R>;
|
execute(args: A, ctx: ToolContext): Promise<R>;
|
||||||
|
|||||||
Reference in new issue
Block a user