584 lines
18 KiB
TypeScript
584 lines
18 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { z } from "zod";
|
|
|
|
import { runWithTools } from "./tool-loop.js";
|
|
import type { Tool } from "@tools/tool";
|
|
import { ToolRegistry } from "@tools/registry";
|
|
import { createToolSearchTool } from "@tools/tool-search";
|
|
|
|
describe("runWithTools", () => {
|
|
it("executes a tool call and returns the model's final answer", async () => {
|
|
const calls: Array<{
|
|
content: string;
|
|
messages: Array<{ role: string; content: string }>;
|
|
options?: Record<string, unknown> | undefined;
|
|
}> = [];
|
|
|
|
const llmClientStub = {
|
|
chatCompletion: (
|
|
messages: Array<{ role: string; content: string }>,
|
|
options?: Record<string, unknown>,
|
|
) => {
|
|
calls.push({
|
|
content: messages[messages.length - 1]?.content ?? "",
|
|
messages,
|
|
options,
|
|
});
|
|
|
|
if (calls.length === 1) {
|
|
return Promise.resolve({
|
|
content:
|
|
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: {
|
|
choices: [
|
|
{
|
|
message: {
|
|
content:
|
|
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: {
|
|
choices: [
|
|
{
|
|
message: {
|
|
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
message: {
|
|
role: "assistant",
|
|
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
|
},
|
|
});
|
|
},
|
|
};
|
|
|
|
const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = {
|
|
name: "validate-nickname",
|
|
description: "닉네임 형식 검사",
|
|
argsSchema: z.object({
|
|
nickname: z.string(),
|
|
}),
|
|
resultSchema: z.object({
|
|
valid: z.boolean(),
|
|
issues: z.array(z.string()),
|
|
}),
|
|
execute: () =>
|
|
Promise.resolve({
|
|
valid: false,
|
|
issues: ["티어는 '/'로 구분해서 넣어주세요."],
|
|
}),
|
|
};
|
|
|
|
const result = await runWithTools({
|
|
llmClient: llmClientStub as never,
|
|
messages: [
|
|
{
|
|
role: "system",
|
|
content: "도구를 필요할 때만 사용해.",
|
|
},
|
|
{
|
|
role: "user",
|
|
content: "닉네임이 왜 안 돼요?",
|
|
},
|
|
],
|
|
toolContext: {},
|
|
tools: [tool],
|
|
});
|
|
|
|
expect(result).toContain("프리리/95/여/D");
|
|
expect(calls[0]?.options?.tools).toHaveLength(1);
|
|
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
|
|
});
|
|
|
|
it("expands searched tools for the next tool-call hop", async () => {
|
|
const toolOptionNames: string[][] = [];
|
|
const llmClientStub = {
|
|
chatCompletion: (
|
|
_messages: Array<Record<string, unknown>>,
|
|
options?: { tools?: Array<{ name: string }> },
|
|
) => {
|
|
toolOptionNames.push(options?.tools?.map((tool) => tool.name) ?? []);
|
|
|
|
if (toolOptionNames.length === 1) {
|
|
return Promise.resolve({
|
|
content:
|
|
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
if (toolOptionNames.length === 2) {
|
|
return Promise.resolve({
|
|
content:
|
|
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: "닉네임 형식을 다시 맞춰주세요.",
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content: "닉네임 형식을 다시 맞춰주세요.",
|
|
},
|
|
});
|
|
},
|
|
};
|
|
const validateTool: Tool<{ nickname: string }, { valid: boolean }> = {
|
|
argsSchema: z.object({
|
|
nickname: z.string(),
|
|
}),
|
|
description: "닉네임 형식을 검사한다.",
|
|
execute: () => Promise.resolve({ valid: false }),
|
|
name: "validate-nickname",
|
|
resultSchema: z.object({
|
|
valid: z.boolean(),
|
|
}),
|
|
};
|
|
const registry = new ToolRegistry().register(validateTool);
|
|
const result = await runWithTools({
|
|
llmClient: llmClientStub as never,
|
|
messages: [{ role: "user", content: "닉네임 확인해줘" }],
|
|
toolContext: { toolRegistry: registry },
|
|
tools: [createToolSearchTool(registry)],
|
|
});
|
|
|
|
expect(result).toContain("닉네임");
|
|
expect(toolOptionNames[0]).toEqual(["search_tools"]);
|
|
expect(toolOptionNames[1]).toEqual(["search_tools", "validate-nickname"]);
|
|
});
|
|
|
|
it("preserves native tool call responses for follow-up turns", async () => {
|
|
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
|
|
|
const llmClientStub = {
|
|
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
|
seenMessages.push(messages);
|
|
|
|
if (seenMessages.length === 1) {
|
|
return Promise.resolve({
|
|
content: "",
|
|
endpoint: "mock://cliproxy",
|
|
provider: "cliproxy",
|
|
raw: {
|
|
choices: [
|
|
{
|
|
message: {
|
|
content: null,
|
|
tool_calls: [
|
|
{
|
|
id: "call_1",
|
|
type: "function",
|
|
function: {
|
|
name: "lookup-community-rule",
|
|
arguments: '{"topic":"age"}',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
],
|
|
},
|
|
message: {
|
|
role: "assistant",
|
|
content: "",
|
|
tool_calls: [
|
|
{
|
|
id: "call_1",
|
|
type: "function",
|
|
function: {
|
|
name: "lookup-community-rule",
|
|
arguments: '{"topic":"age"}',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: "나이 기준은 27세 이상이에요.",
|
|
endpoint: "mock://cliproxy",
|
|
provider: "cliproxy",
|
|
raw: {
|
|
choices: [
|
|
{
|
|
message: {
|
|
content: "나이 기준은 27세 이상이에요.",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
message: {
|
|
role: "assistant",
|
|
content: "나이 기준은 27세 이상이에요.",
|
|
},
|
|
});
|
|
},
|
|
};
|
|
|
|
const tool: Tool<{ topic: string }, { topic: string; text: string }> = {
|
|
name: "lookup-community-rule",
|
|
description: "커뮤니티 규칙 조회",
|
|
argsSchema: z.object({
|
|
topic: z.string(),
|
|
}),
|
|
resultSchema: z.object({
|
|
topic: z.string(),
|
|
text: z.string(),
|
|
}),
|
|
execute: () =>
|
|
Promise.resolve({
|
|
topic: "age",
|
|
text: "나이 기준은 27세 이상이에요.",
|
|
}),
|
|
};
|
|
|
|
const result = await runWithTools({
|
|
llmClient: llmClientStub as never,
|
|
messages: [
|
|
{
|
|
role: "system",
|
|
content: "필요할 때만 도구를 사용해.",
|
|
},
|
|
{
|
|
role: "user",
|
|
content: "나이 기준이 뭐예요?",
|
|
},
|
|
],
|
|
toolContext: {},
|
|
tools: [tool],
|
|
});
|
|
|
|
expect(result).toBe("나이 기준은 27세 이상이에요.");
|
|
expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true);
|
|
expect(
|
|
seenMessages[1]?.some(
|
|
(message) => Array.isArray(message.tool_calls) && message.role === "assistant",
|
|
),
|
|
).toBe(true);
|
|
});
|
|
|
|
it("returns permanent tool failures to the model instead of throwing", async () => {
|
|
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
|
const llmClientStub = {
|
|
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
|
seenMessages.push(messages);
|
|
|
|
if (seenMessages.length === 1) {
|
|
return Promise.resolve({
|
|
content:
|
|
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
|
|
},
|
|
});
|
|
},
|
|
};
|
|
|
|
const tool: Tool<{ value: string }, { ok: boolean }> = {
|
|
name: "explode",
|
|
description: "항상 실패",
|
|
argsSchema: z.object({
|
|
value: z.string(),
|
|
}),
|
|
resultSchema: z.object({
|
|
ok: z.boolean(),
|
|
}),
|
|
execute: () => Promise.reject(new Error("boom")),
|
|
};
|
|
|
|
const result = await runWithTools({
|
|
llmClient: llmClientStub as never,
|
|
messages: [{ role: "user", content: "확인해줘" }],
|
|
toolContext: {},
|
|
tools: [tool],
|
|
});
|
|
|
|
expect(result).toContain("기본 안내");
|
|
const toolMessage = seenMessages[1]?.find(
|
|
(message) => message.role === "tool",
|
|
);
|
|
expect(JSON.parse(String(toolMessage?.content))).toMatchObject({
|
|
errorKind: "permanent",
|
|
});
|
|
});
|
|
|
|
it("allows one self-correction hop when tool arguments are invalid", async () => {
|
|
let executeCount = 0;
|
|
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
|
const llmClientStub = {
|
|
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
|
seenMessages.push(messages);
|
|
|
|
if (seenMessages.length === 1) {
|
|
return Promise.resolve({
|
|
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
if (seenMessages.length === 2) {
|
|
return Promise.resolve({
|
|
content:
|
|
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: "프리리님 기준으로 확인했어요.",
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content: "프리리님 기준으로 확인했어요.",
|
|
},
|
|
});
|
|
},
|
|
};
|
|
|
|
const tool: Tool<{ name: string }, { ok: boolean }> = {
|
|
name: "needs-name",
|
|
description: "이름 필요",
|
|
argsSchema: z.object({
|
|
name: z.string(),
|
|
}),
|
|
resultSchema: z.object({
|
|
ok: z.boolean(),
|
|
}),
|
|
execute: () => {
|
|
executeCount += 1;
|
|
return Promise.resolve({ ok: true });
|
|
},
|
|
};
|
|
|
|
const result = await runWithTools({
|
|
llmClient: llmClientStub as never,
|
|
messages: [{ role: "user", content: "확인해줘" }],
|
|
toolContext: {},
|
|
tools: [tool],
|
|
});
|
|
|
|
expect(result).toContain("프리리님");
|
|
expect(executeCount).toBe(1);
|
|
expect(JSON.stringify(seenMessages[1])).toContain("schema");
|
|
});
|
|
|
|
it("retries transient tool errors once before continuing", async () => {
|
|
let executeCount = 0;
|
|
const llmClientStub = {
|
|
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
|
if (messages.some((message) => message.role === "tool")) {
|
|
return Promise.resolve({
|
|
content: "재시도 후 확인됐어요.",
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content: "재시도 후 확인됐어요.",
|
|
},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: '<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
|
|
},
|
|
});
|
|
},
|
|
};
|
|
|
|
const tool: Tool<{ value: string }, { ok: boolean }> = {
|
|
name: "flaky",
|
|
description: "일시 실패",
|
|
argsSchema: z.object({
|
|
value: z.string(),
|
|
}),
|
|
resultSchema: z.object({
|
|
ok: z.boolean(),
|
|
}),
|
|
execute: () => {
|
|
executeCount += 1;
|
|
if (executeCount === 1) {
|
|
const error = new Error("timeout while calling dependency");
|
|
error.name = "AbortError";
|
|
return Promise.reject(error);
|
|
}
|
|
|
|
return Promise.resolve({ ok: true });
|
|
},
|
|
};
|
|
|
|
const result = await runWithTools({
|
|
llmClient: llmClientStub as never,
|
|
messages: [{ role: "user", content: "확인해줘" }],
|
|
toolContext: {},
|
|
tools: [tool],
|
|
});
|
|
|
|
expect(result).toBe("재시도 후 확인됐어요.");
|
|
expect(executeCount).toBe(2);
|
|
});
|
|
|
|
it("grants one extra hop for precondition failures", async () => {
|
|
let executeCount = 0;
|
|
const llmClientStub = {
|
|
chatCompletion: () => {
|
|
if (executeCount === 0) {
|
|
return Promise.resolve({
|
|
content:
|
|
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
if (executeCount === 1) {
|
|
return Promise.resolve({
|
|
content:
|
|
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content:
|
|
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
|
|
},
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({
|
|
content: "추가 확인까지 마쳤어요.",
|
|
endpoint: "mock://llm",
|
|
provider: "llama",
|
|
raw: { choices: [{ message: { content: "" } }] },
|
|
message: {
|
|
role: "assistant",
|
|
content: "추가 확인까지 마쳤어요.",
|
|
},
|
|
});
|
|
},
|
|
};
|
|
|
|
const tool: Tool<
|
|
{ value: string },
|
|
{ ok: boolean; errorKind?: string | undefined; message?: string | undefined }
|
|
> = {
|
|
name: "needs-session",
|
|
description: "세션 필요",
|
|
argsSchema: z.object({
|
|
value: z.string(),
|
|
}),
|
|
resultSchema: z.object({
|
|
ok: z.boolean(),
|
|
errorKind: z.string().optional(),
|
|
message: z.string().optional(),
|
|
}),
|
|
execute: () => {
|
|
executeCount += 1;
|
|
if (executeCount === 1) {
|
|
return Promise.resolve({
|
|
ok: false,
|
|
errorKind: "precondition-failed",
|
|
message: "세션이 없어요.",
|
|
});
|
|
}
|
|
|
|
return Promise.resolve({ ok: true });
|
|
},
|
|
};
|
|
|
|
const result = await runWithTools({
|
|
llmClient: llmClientStub as never,
|
|
maxHops: 1,
|
|
messages: [{ role: "user", content: "확인해줘" }],
|
|
toolContext: {},
|
|
tools: [tool],
|
|
});
|
|
|
|
expect(result).toContain("추가 확인");
|
|
expect(executeCount).toBe(2);
|
|
});
|
|
});
|