Add skill governance and runtime routing
This commit is contained in:
1 parent
6f09408423
commit
41dcfcdef7
71 files changed
+7057
-102
No files matched your search
@@ -0,0 +1,417 @@
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import { withToolSpan } from "@obs/spans";
|
||||
import type {
|
||||
Tool,
|
||||
ToolContext,
|
||||
ToolErrorKind,
|
||||
ToolExecutionFailure,
|
||||
} from "@tools/tool";
|
||||
import type { ToolPlan, ToolPlanStep } from "./planner-schema.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface ToolPlanResult {
|
||||
durationMs: number;
|
||||
failures: ToolExecutionFailure[];
|
||||
fallbackToLlm: boolean;
|
||||
results: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface OrderedPlan {
|
||||
failures: ToolExecutionFailure[];
|
||||
levels: ToolPlanStep[][];
|
||||
}
|
||||
|
||||
function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isTransientError(error: unknown): boolean {
|
||||
const name = error instanceof Error ? error.name : "";
|
||||
const message = getErrorMessage(error);
|
||||
return (
|
||||
name === "AbortError" ||
|
||||
name === "FetchError" ||
|
||||
/(?:timeout|timed out|ECONNRESET|ECONNREFUSED|EAI_AGAIN|ETIMEDOUT)/iu.test(
|
||||
message,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function classifyError(toolName: string, error: unknown): ToolExecutionFailure {
|
||||
const message = getErrorMessage(error);
|
||||
const isArgsInvalid =
|
||||
error instanceof Error &&
|
||||
(error.name === "ZodError" || error.constructor.name === "ZodError");
|
||||
|
||||
if (isArgsInvalid) {
|
||||
return {
|
||||
kind: "args-invalid",
|
||||
message,
|
||||
retryable: true,
|
||||
toolName,
|
||||
details: error,
|
||||
};
|
||||
}
|
||||
|
||||
if (isTransientError(error)) {
|
||||
return {
|
||||
kind: "transient",
|
||||
message,
|
||||
retryable: true,
|
||||
toolName,
|
||||
details: error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "permanent",
|
||||
message,
|
||||
retryable: false,
|
||||
toolName,
|
||||
details: error,
|
||||
};
|
||||
}
|
||||
|
||||
function makeFailure(input: {
|
||||
details?: unknown;
|
||||
kind: ToolErrorKind;
|
||||
message: string;
|
||||
retryable?: boolean;
|
||||
toolName: string;
|
||||
}): ToolExecutionFailure {
|
||||
return {
|
||||
kind: input.kind,
|
||||
message: input.message,
|
||||
retryable: input.retryable ?? false,
|
||||
toolName: input.toolName,
|
||||
...(input.details === undefined ? {} : { details: input.details }),
|
||||
};
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function isToolErrorKind(value: unknown): value is ToolErrorKind {
|
||||
return (
|
||||
value === "args-invalid" ||
|
||||
value === "transient" ||
|
||||
value === "precondition-failed" ||
|
||||
value === "permanent"
|
||||
);
|
||||
}
|
||||
|
||||
function getStructuredFailure(
|
||||
step: ToolPlanStep,
|
||||
result: unknown,
|
||||
): ToolExecutionFailure | null {
|
||||
const record = asRecord(result);
|
||||
if (record["ok"] !== false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const kind = isToolErrorKind(record["errorKind"])
|
||||
? record["errorKind"]
|
||||
: isToolErrorKind(record["kind"])
|
||||
? record["kind"]
|
||||
: "precondition-failed";
|
||||
const message =
|
||||
typeof record["message"] === "string" && record["message"].trim()
|
||||
? record["message"]
|
||||
: "Tool precondition failed";
|
||||
|
||||
return makeFailure({
|
||||
details: {
|
||||
result,
|
||||
stepId: step.id,
|
||||
},
|
||||
kind,
|
||||
message,
|
||||
retryable: kind === "args-invalid" || kind === "transient",
|
||||
toolName: step.tool,
|
||||
});
|
||||
}
|
||||
|
||||
function parsePath(path: string): string[] {
|
||||
const normalized = path.trim().replace(/^\$/u, "");
|
||||
return normalized
|
||||
.split(".")
|
||||
.flatMap((segment) => {
|
||||
const bracketIndex = segment.indexOf("[");
|
||||
if (bracketIndex === -1 || !segment.endsWith("]")) {
|
||||
return [segment];
|
||||
}
|
||||
|
||||
const key = segment.slice(0, bracketIndex);
|
||||
const index = segment.slice(bracketIndex + 1, -1);
|
||||
return key && index ? [key, index] : [segment];
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function readPath(root: unknown, path: string): unknown {
|
||||
let current = root;
|
||||
for (const segment of parsePath(path)) {
|
||||
if (Array.isArray(current)) {
|
||||
const index = Number(segment);
|
||||
if (!Number.isInteger(index)) {
|
||||
throw new Error(`Invalid array index in path: ${path}`);
|
||||
}
|
||||
current = current[index];
|
||||
continue;
|
||||
}
|
||||
|
||||
const record = asRecord(current);
|
||||
if (!(segment in record)) {
|
||||
throw new Error(`Missing context path: ${path}`);
|
||||
}
|
||||
current = record[segment];
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
function buildArgs(input: {
|
||||
ctx: ToolContext;
|
||||
request: AgentRequest;
|
||||
results: Record<string, unknown>;
|
||||
step: ToolPlanStep;
|
||||
}): Record<string, unknown> {
|
||||
const args: Record<string, unknown> = {
|
||||
...(input.step.argsLiteral ?? {}),
|
||||
};
|
||||
const roots = {
|
||||
context: input.ctx,
|
||||
request: input.request,
|
||||
results: input.results,
|
||||
session: input.ctx.session ?? null,
|
||||
tool: input.results,
|
||||
};
|
||||
|
||||
for (const [targetKey, sourcePath] of Object.entries(
|
||||
input.step.argsFromContext ?? {},
|
||||
)) {
|
||||
args[targetKey] = readPath(roots, sourcePath);
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function makePlanFailure(message: string, details?: unknown): ToolExecutionFailure {
|
||||
return makeFailure({
|
||||
details,
|
||||
kind: "permanent",
|
||||
message,
|
||||
toolName: "tool-plan",
|
||||
});
|
||||
}
|
||||
|
||||
function orderPlan(plan: ToolPlan): OrderedPlan {
|
||||
const failures: ToolExecutionFailure[] = [];
|
||||
const byId = new Map<string, ToolPlanStep>();
|
||||
for (const step of plan.steps) {
|
||||
if (byId.has(step.id)) {
|
||||
failures.push(makePlanFailure(`Duplicate toolPlan step id: ${step.id}`));
|
||||
continue;
|
||||
}
|
||||
byId.set(step.id, step);
|
||||
}
|
||||
|
||||
for (const step of plan.steps) {
|
||||
for (const dependency of step.dependsOn ?? []) {
|
||||
if (!byId.has(dependency)) {
|
||||
failures.push(
|
||||
makePlanFailure(`Missing dependency ${dependency} for step ${step.id}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
return { failures, levels: [] };
|
||||
}
|
||||
|
||||
const remaining = new Map(byId);
|
||||
const completed = new Set<string>();
|
||||
const levels: ToolPlanStep[][] = [];
|
||||
|
||||
while (remaining.size > 0) {
|
||||
const ready = [...remaining.values()].filter((step) =>
|
||||
(step.dependsOn ?? []).every((dependency) => completed.has(dependency)),
|
||||
);
|
||||
|
||||
if (ready.length === 0) {
|
||||
return {
|
||||
failures: [makePlanFailure("Cyclic toolPlan dependencies detected")],
|
||||
levels,
|
||||
};
|
||||
}
|
||||
|
||||
levels.push(ready);
|
||||
for (const step of ready) {
|
||||
remaining.delete(step.id);
|
||||
completed.add(step.id);
|
||||
}
|
||||
}
|
||||
|
||||
return { failures, levels };
|
||||
}
|
||||
|
||||
async function runStep(input: {
|
||||
ctx: ToolContext;
|
||||
request: AgentRequest;
|
||||
results: Record<string, unknown>;
|
||||
step: ToolPlanStep;
|
||||
tools: readonly Tool<unknown, unknown>[];
|
||||
}): Promise<ToolExecutionFailure | null> {
|
||||
const tool = input.tools.find((candidate) => candidate.name === input.step.tool);
|
||||
if (!tool) {
|
||||
return makeFailure({
|
||||
details: { stepId: input.step.id },
|
||||
kind: "permanent",
|
||||
message: `Tool not registered for step: ${input.step.tool}`,
|
||||
toolName: input.step.tool,
|
||||
});
|
||||
}
|
||||
|
||||
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
||||
let args: Record<string, unknown> = {};
|
||||
try {
|
||||
args = buildArgs(input);
|
||||
const validated = tool.argsSchema.parse(args);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.name": tool.name,
|
||||
"tool.plan.step_id": input.step.id,
|
||||
},
|
||||
() => tool.execute(validated, input.ctx),
|
||||
);
|
||||
const structuredFailure = getStructuredFailure(input.step, result);
|
||||
if (structuredFailure) {
|
||||
traceStep?.finish({
|
||||
args,
|
||||
errorKind: structuredFailure.kind,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: structuredFailure.message,
|
||||
});
|
||||
return structuredFailure;
|
||||
}
|
||||
|
||||
input.results[input.step.id] = result;
|
||||
traceStep?.finish({
|
||||
args,
|
||||
name: tool.name,
|
||||
ok: true,
|
||||
resultSummary: summarizeToolResult(result),
|
||||
});
|
||||
return null;
|
||||
} catch (error) {
|
||||
const failure = classifyError(tool.name, error);
|
||||
traceStep?.finish({
|
||||
args,
|
||||
errorKind: failure.kind,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: failure.message,
|
||||
});
|
||||
return failure;
|
||||
}
|
||||
}
|
||||
|
||||
function shouldFallback(step: ToolPlanStep): boolean {
|
||||
return step.on?.error === "fallback-llm";
|
||||
}
|
||||
|
||||
async function runLevel(input: {
|
||||
ctx: ToolContext;
|
||||
failures: ToolExecutionFailure[];
|
||||
request: AgentRequest;
|
||||
results: Record<string, unknown>;
|
||||
steps: ToolPlanStep[];
|
||||
tools: readonly Tool<unknown, unknown>[];
|
||||
}): Promise<{ abort: boolean; fallbackToLlm: boolean }> {
|
||||
let abort = false;
|
||||
let fallbackToLlm = false;
|
||||
|
||||
for (let index = 0; index < input.steps.length; index += 2) {
|
||||
const chunk = input.steps.slice(index, index + 2);
|
||||
const failures = await Promise.all(
|
||||
chunk.map((step) =>
|
||||
runStep({
|
||||
ctx: input.ctx,
|
||||
request: input.request,
|
||||
results: input.results,
|
||||
step,
|
||||
tools: input.tools,
|
||||
}).then((failure) => ({ failure, step })),
|
||||
),
|
||||
);
|
||||
|
||||
for (const { failure, step } of failures) {
|
||||
if (!failure) {
|
||||
continue;
|
||||
}
|
||||
input.failures.push(failure);
|
||||
if (shouldFallback(step)) {
|
||||
fallbackToLlm = true;
|
||||
}
|
||||
if (step.on?.error === "abort" || !step.on?.error) {
|
||||
abort = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (abort || fallbackToLlm) {
|
||||
return { abort, fallbackToLlm };
|
||||
}
|
||||
}
|
||||
|
||||
return { abort, fallbackToLlm };
|
||||
}
|
||||
|
||||
export async function runToolPlan(
|
||||
plan: ToolPlan,
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
ctx: ToolContext,
|
||||
request: AgentRequest,
|
||||
): Promise<ToolPlanResult> {
|
||||
const startedAt = Date.now();
|
||||
const results: Record<string, unknown> = {};
|
||||
const ordered = orderPlan(plan);
|
||||
const failures = [...ordered.failures];
|
||||
let fallbackToLlm = false;
|
||||
|
||||
for (const level of ordered.levels) {
|
||||
const levelResult = await runLevel({
|
||||
ctx,
|
||||
failures,
|
||||
request,
|
||||
results,
|
||||
steps: level,
|
||||
tools,
|
||||
});
|
||||
|
||||
fallbackToLlm = fallbackToLlm || levelResult.fallbackToLlm;
|
||||
if (levelResult.abort || levelResult.fallbackToLlm) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
durationMs: Math.max(0, Date.now() - startedAt),
|
||||
failures,
|
||||
fallbackToLlm,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeToolResult(result: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(result).slice(0, 200);
|
||||
} catch {
|
||||
return String(result).slice(0, 200);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user