Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit 6f09408423
57 files changed
+7060 -142

No files matched your search

+36 -9
View File
@@ -4,6 +4,7 @@ import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
import type { Tool, ToolContext } from "@tools/tool";
import { getActiveProcessTrace } from "./trace.js";
export interface ToolLoopOptions {
deadlineMs?: number | undefined;
@@ -107,15 +108,33 @@ export async function runWithTools(
"[ToolLoop] Executing tool call",
);
const validated = tool.argsSchema.parse(call.arguments);
const result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
let result: unknown;
try {
const validated = tool.argsSchema.parse(call.arguments);
result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
traceStep?.finish({
args: call.arguments,
name: tool.name,
ok: true,
resultSummary: summarizeToolResult(result),
});
} catch (error) {
traceStep?.finish({
args: call.arguments,
name: tool.name,
ok: false,
resultSummary: error instanceof Error ? error.message : String(error),
});
throw error;
}
logger.debug(
{
@@ -137,3 +156,11 @@ export async function runWithTools(
return null;
}
function summarizeToolResult(result: unknown): string {
try {
return JSON.stringify(result).slice(0, 200);
} catch {
return String(result).slice(0, 200);
}
}