import { describe, expect, it } from "vitest"; import { z } from "zod"; import { runWithTools } from "./tool-loop.js"; import type { Tool } from "@tools/tool"; import { ToolRegistry } from "@tools/registry"; import { createToolSearchTool } from "@tools/tool-search"; describe("runWithTools", () => { it("executes a tool call and returns the model's final answer", async () => { const calls: Array<{ content: string; messages: Array<{ role: string; content: string }>; options?: Record | undefined; }> = []; const llmClientStub = { chatCompletion: ( messages: Array<{ role: string; content: string }>, options?: Record, ) => { calls.push({ content: messages[messages.length - 1]?.content ?? "", messages, options, }); if (calls.length === 1) { return Promise.resolve({ content: '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [ { message: { content: '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', }, }, ], }, message: { role: "assistant", content: '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', }, }); } return Promise.resolve({ content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D", endpoint: "mock://llm", provider: "llama", raw: { choices: [ { message: { content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D", }, }, ], }, message: { role: "assistant", content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D", }, }); }, }; const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = { name: "validate-nickname", description: "닉네임 형식 검사", argsSchema: z.object({ nickname: z.string(), }), resultSchema: z.object({ valid: z.boolean(), issues: z.array(z.string()), }), execute: () => Promise.resolve({ valid: false, issues: ["티어는 '/'로 구분해서 넣어주세요."], }), }; const result = await runWithTools({ llmClient: llmClientStub as never, messages: [ { role: "system", content: "도구를 필요할 때만 사용해.", }, { role: "user", content: "닉네임이 왜 안 돼요?", }, ], toolContext: {}, tools: [tool], }); expect(result).toContain("프리리/95/여/D"); expect(calls[0]?.options?.tools).toHaveLength(1); expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true); }); it("expands searched tools for the next tool-call hop", async () => { const toolOptionNames: string[][] = []; const llmClientStub = { chatCompletion: ( _messages: Array>, options?: { tools?: Array<{ name: string }> }, ) => { toolOptionNames.push(options?.tools?.map((tool) => tool.name) ?? []); if (toolOptionNames.length === 1) { return Promise.resolve({ content: '{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}', }, }); } if (toolOptionNames.length === 2) { return Promise.resolve({ content: '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', }, }); } return Promise.resolve({ content: "닉네임 형식을 다시 맞춰주세요.", endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: "닉네임 형식을 다시 맞춰주세요.", }, }); }, }; const validateTool: Tool<{ nickname: string }, { valid: boolean }> = { argsSchema: z.object({ nickname: z.string(), }), description: "닉네임 형식을 검사한다.", execute: () => Promise.resolve({ valid: false }), name: "validate-nickname", resultSchema: z.object({ valid: z.boolean(), }), }; const registry = new ToolRegistry().register(validateTool); const result = await runWithTools({ llmClient: llmClientStub as never, messages: [{ role: "user", content: "닉네임 확인해줘" }], toolContext: { toolRegistry: registry }, tools: [createToolSearchTool(registry)], }); expect(result).toContain("닉네임"); expect(toolOptionNames[0]).toEqual(["search_tools"]); expect(toolOptionNames[1]).toEqual(["search_tools", "validate-nickname"]); }); it("preserves native tool call responses for follow-up turns", async () => { const seenMessages: Array>> = []; const llmClientStub = { chatCompletion: (messages: Array>) => { seenMessages.push(messages); if (seenMessages.length === 1) { return Promise.resolve({ content: "", endpoint: "mock://cliproxy", provider: "cliproxy", raw: { choices: [ { message: { content: null, tool_calls: [ { id: "call_1", type: "function", function: { name: "lookup-community-rule", arguments: '{"topic":"age"}', }, }, ], }, }, ], }, message: { role: "assistant", content: "", tool_calls: [ { id: "call_1", type: "function", function: { name: "lookup-community-rule", arguments: '{"topic":"age"}', }, }, ], }, }); } return Promise.resolve({ content: "나이 기준은 27세 이상이에요.", endpoint: "mock://cliproxy", provider: "cliproxy", raw: { choices: [ { message: { content: "나이 기준은 27세 이상이에요.", }, }, ], }, message: { role: "assistant", content: "나이 기준은 27세 이상이에요.", }, }); }, }; const tool: Tool<{ topic: string }, { topic: string; text: string }> = { name: "lookup-community-rule", description: "커뮤니티 규칙 조회", argsSchema: z.object({ topic: z.string(), }), resultSchema: z.object({ topic: z.string(), text: z.string(), }), execute: () => Promise.resolve({ topic: "age", text: "나이 기준은 27세 이상이에요.", }), }; const result = await runWithTools({ llmClient: llmClientStub as never, messages: [ { role: "system", content: "필요할 때만 도구를 사용해.", }, { role: "user", content: "나이 기준이 뭐예요?", }, ], toolContext: {}, tools: [tool], }); expect(result).toBe("나이 기준은 27세 이상이에요."); expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true); expect( seenMessages[1]?.some( (message) => Array.isArray(message.tool_calls) && message.role === "assistant", ), ).toBe(true); }); it("returns permanent tool failures to the model instead of throwing", async () => { const seenMessages: Array>> = []; const llmClientStub = { chatCompletion: (messages: Array>) => { seenMessages.push(messages); if (seenMessages.length === 1) { return Promise.resolve({ content: '{"name":"explode","arguments":{"value":"x"}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"explode","arguments":{"value":"x"}}', }, }); } return Promise.resolve({ content: "지금은 확인이 어려워서 기본 안내만 드릴게요.", endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: "지금은 확인이 어려워서 기본 안내만 드릴게요.", }, }); }, }; const tool: Tool<{ value: string }, { ok: boolean }> = { name: "explode", description: "항상 실패", argsSchema: z.object({ value: z.string(), }), resultSchema: z.object({ ok: z.boolean(), }), execute: () => Promise.reject(new Error("boom")), }; const result = await runWithTools({ llmClient: llmClientStub as never, messages: [{ role: "user", content: "확인해줘" }], toolContext: {}, tools: [tool], }); expect(result).toContain("기본 안내"); const toolMessage = seenMessages[1]?.find( (message) => message.role === "tool", ); expect(JSON.parse(String(toolMessage?.content))).toMatchObject({ errorKind: "permanent", }); }); it("allows one self-correction hop when tool arguments are invalid", async () => { let executeCount = 0; const seenMessages: Array>> = []; const llmClientStub = { chatCompletion: (messages: Array>) => { seenMessages.push(messages); if (seenMessages.length === 1) { return Promise.resolve({ content: '{"name":"needs-name","arguments":{}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"needs-name","arguments":{}}', }, }); } if (seenMessages.length === 2) { return Promise.resolve({ content: '{"name":"needs-name","arguments":{"name":"프리리"}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"needs-name","arguments":{"name":"프리리"}}', }, }); } return Promise.resolve({ content: "프리리님 기준으로 확인했어요.", endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: "프리리님 기준으로 확인했어요.", }, }); }, }; const tool: Tool<{ name: string }, { ok: boolean }> = { name: "needs-name", description: "이름 필요", argsSchema: z.object({ name: z.string(), }), resultSchema: z.object({ ok: z.boolean(), }), execute: () => { executeCount += 1; return Promise.resolve({ ok: true }); }, }; const result = await runWithTools({ llmClient: llmClientStub as never, messages: [{ role: "user", content: "확인해줘" }], toolContext: {}, tools: [tool], }); expect(result).toContain("프리리님"); expect(executeCount).toBe(1); expect(JSON.stringify(seenMessages[1])).toContain("schema"); }); it("retries transient tool errors once before continuing", async () => { let executeCount = 0; const llmClientStub = { chatCompletion: (messages: Array>) => { if (messages.some((message) => message.role === "tool")) { return Promise.resolve({ content: "재시도 후 확인됐어요.", endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: "재시도 후 확인됐어요.", }, }); } return Promise.resolve({ content: '{"name":"flaky","arguments":{"value":"x"}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"flaky","arguments":{"value":"x"}}', }, }); }, }; const tool: Tool<{ value: string }, { ok: boolean }> = { name: "flaky", description: "일시 실패", argsSchema: z.object({ value: z.string(), }), resultSchema: z.object({ ok: z.boolean(), }), execute: () => { executeCount += 1; if (executeCount === 1) { const error = new Error("timeout while calling dependency"); error.name = "AbortError"; return Promise.reject(error); } return Promise.resolve({ ok: true }); }, }; const result = await runWithTools({ llmClient: llmClientStub as never, messages: [{ role: "user", content: "확인해줘" }], toolContext: {}, tools: [tool], }); expect(result).toBe("재시도 후 확인됐어요."); expect(executeCount).toBe(2); }); it("grants one extra hop for precondition failures", async () => { let executeCount = 0; const llmClientStub = { chatCompletion: () => { if (executeCount === 0) { return Promise.resolve({ content: '{"name":"needs-session","arguments":{"value":"first"}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"needs-session","arguments":{"value":"first"}}', }, }); } if (executeCount === 1) { return Promise.resolve({ content: '{"name":"needs-session","arguments":{"value":"second"}}', endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: '{"name":"needs-session","arguments":{"value":"second"}}', }, }); } return Promise.resolve({ content: "추가 확인까지 마쳤어요.", endpoint: "mock://llm", provider: "llama", raw: { choices: [{ message: { content: "" } }] }, message: { role: "assistant", content: "추가 확인까지 마쳤어요.", }, }); }, }; const tool: Tool< { value: string }, { ok: boolean; errorKind?: string | undefined; message?: string | undefined } > = { name: "needs-session", description: "세션 필요", argsSchema: z.object({ value: z.string(), }), resultSchema: z.object({ ok: z.boolean(), errorKind: z.string().optional(), message: z.string().optional(), }), execute: () => { executeCount += 1; if (executeCount === 1) { return Promise.resolve({ ok: false, errorKind: "precondition-failed", message: "세션이 없어요.", }); } return Promise.resolve({ ok: true }); }, }; const result = await runWithTools({ llmClient: llmClientStub as never, maxHops: 1, messages: [{ role: "user", content: "확인해줘" }], toolContext: {}, tools: [tool], }); expect(result).toContain("추가 확인"); expect(executeCount).toBe(2); }); });