Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

@@ -0,0 +1,219 @@
import { describe, expect, it } from "vitest";
import { z } from "zod";
import { runWithTools } from "./tool-loop.js";
import type { Tool } from "@tools/tool";
describe("runWithTools", () => {
it("executes a tool call and returns the model's final answer", async () => {
const calls: Array<{
content: string;
messages: Array<{ role: string; content: string }>;
options?: Record<string, unknown> | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: Array<{ role: string; content: string }>,
options?: Record<string, unknown>,
) => {
calls.push({
content: messages[messages.length - 1]?.content ?? "",
messages,
options,
});
if (calls.length === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: {
choices: [
{
message: {
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
},
],
},
message: {
role: "assistant",
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
endpoint: "mock://llm",
provider: "llama",
raw: {
choices: [
{
message: {
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
},
},
],
},
message: {
role: "assistant",
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
},
});
},
};
const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = {
name: "validate-nickname",
description: "닉네임 형식 검사",
argsSchema: z.object({
nickname: z.string(),
}),
resultSchema: z.object({
valid: z.boolean(),
issues: z.array(z.string()),
}),
execute: () =>
Promise.resolve({
valid: false,
issues: ["티어는 '/'로 구분해서 넣어주세요."],
}),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [
{
role: "system",
content: "도구를 필요할 때만 사용해.",
},
{
role: "user",
content: "닉네임이 왜 안 돼요?",
},
],
toolContext: {},
tools: [tool],
});
expect(result).toContain("프리리/95/여/D");
expect(calls[0]?.options?.tools).toHaveLength(1);
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
});
it("preserves native tool call responses for follow-up turns", async () => {
const seenMessages: Array<Array<Record<string, unknown>>> = [];
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
seenMessages.push(messages);
if (seenMessages.length === 1) {
return Promise.resolve({
content: "",
endpoint: "mock://cliproxy",
provider: "cliproxy",
raw: {
choices: [
{
message: {
content: null,
tool_calls: [
{
id: "call_1",
type: "function",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
},
},
],
},
message: {
role: "assistant",
content: "",
tool_calls: [
{
id: "call_1",
type: "function",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
},
});
}
return Promise.resolve({
content: "나이 기준은 27세 이상이에요.",
endpoint: "mock://cliproxy",
provider: "cliproxy",
raw: {
choices: [
{
message: {
content: "나이 기준은 27세 이상이에요.",
},
},
],
},
message: {
role: "assistant",
content: "나이 기준은 27세 이상이에요.",
},
});
},
};
const tool: Tool<{ topic: string }, { topic: string; text: string }> = {
name: "lookup-community-rule",
description: "커뮤니티 규칙 조회",
argsSchema: z.object({
topic: z.string(),
}),
resultSchema: z.object({
topic: z.string(),
text: z.string(),
}),
execute: () =>
Promise.resolve({
topic: "age",
text: "나이 기준은 27세 이상이에요.",
}),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [
{
role: "system",
content: "필요할 때만 도구를 사용해.",
},
{
role: "user",
content: "나이 기준이 뭐예요?",
},
],
toolContext: {},
tools: [tool],
});
expect(result).toBe("나이 기준은 27세 이상이에요.");
expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true);
expect(
seenMessages[1]?.some(
(message) => Array.isArray(message.tool_calls) && message.role === "assistant",
),
).toBe(true);
});
});