Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

@@ -0,0 +1,89 @@
import { withToolSpan } from "@obs/spans";
import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
import type { Tool, ToolContext } from "@tools/tool";
export interface ToolLoopOptions {
deadlineMs?: number | undefined;
llmClient: LlmClient;
maxHops?: number | undefined;
maxTokens?: number | undefined;
messages: ChatMessage[];
modelOverride?: string | undefined;
role?: ChatCompletionOptions["role"];
signal?: AbortSignal | undefined;
temperature?: number | undefined;
tier?: ChatCompletionOptions["tier"];
toolContext: ToolContext;
tools: readonly Tool<unknown, unknown>[];
}
export async function runWithTools(
options: ToolLoopOptions,
): Promise<string | null> {
const maxHops = options.maxHops ?? 2;
const messages = options.messages.map((message) => ({ ...message }));
let allowNativeTools = true;
for (let hop = 0; hop <= maxHops; hop++) {
const response = await options.llmClient.chatCompletion(messages, {
deadlineMs: options.deadlineMs,
maxTokens: options.maxTokens,
modelOverride: options.modelOverride,
role: options.role,
signal: options.signal,
temperature: options.temperature,
tier: options.tier,
tools: allowNativeTools ? options.tools : undefined,
});
if (!response) {
return null;
}
const toolCalls = parseToolCalls(response.message);
if (toolCalls.length === 0) {
return response.content;
}
if (hop === maxHops) {
messages.push(response.message);
messages.push({
role: "user",
content:
"도구 호출은 여기까지 하고, 지금까지 얻은 정보만으로 한국어 문장으로 최종 답변해줘.",
});
allowNativeTools = false;
continue;
}
messages.push(response.message);
for (const call of toolCalls) {
const tool = options.tools.find((candidate) => candidate.name === call.name);
if (!tool) {
throw new Error(`Tool not registered for call: ${call.name}`);
}
const validated = tool.argsSchema.parse(call.arguments);
const result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
messages.push({
role: "tool",
name: tool.name,
tool_call_id: call.id,
content: JSON.stringify(result),
});
}
}
return null;
}