Add skill governance and runtime routing
This commit is contained in:
1 parent
a900016be4
commit
221e9391e0
71 files changed
+7057
-102
No files matched your search
@@ -3,6 +3,8 @@ import { z } from "zod";
|
||||
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
import type { Tool } from "@tools/tool";
|
||||
import { ToolRegistry } from "@tools/registry";
|
||||
import { createToolSearchTool } from "@tools/tool-search";
|
||||
|
||||
describe("runWithTools", () => {
|
||||
it("executes a tool call and returns the model's final answer", async () => {
|
||||
@@ -106,6 +108,81 @@ describe("runWithTools", () => {
|
||||
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
|
||||
});
|
||||
|
||||
it("expands searched tools for the next tool-call hop", async () => {
|
||||
const toolOptionNames: string[][] = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
_messages: Array<Record<string, unknown>>,
|
||||
options?: { tools?: Array<{ name: string }> },
|
||||
) => {
|
||||
toolOptionNames.push(options?.tools?.map((tool) => tool.name) ?? []);
|
||||
|
||||
if (toolOptionNames.length === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (toolOptionNames.length === 2) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "닉네임 형식을 다시 맞춰주세요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "닉네임 형식을 다시 맞춰주세요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
const validateTool: Tool<{ nickname: string }, { valid: boolean }> = {
|
||||
argsSchema: z.object({
|
||||
nickname: z.string(),
|
||||
}),
|
||||
description: "닉네임 형식을 검사한다.",
|
||||
execute: () => Promise.resolve({ valid: false }),
|
||||
name: "validate-nickname",
|
||||
resultSchema: z.object({
|
||||
valid: z.boolean(),
|
||||
}),
|
||||
};
|
||||
const registry = new ToolRegistry().register(validateTool);
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "닉네임 확인해줘" }],
|
||||
toolContext: { toolRegistry: registry },
|
||||
tools: [createToolSearchTool(registry)],
|
||||
});
|
||||
|
||||
expect(result).toContain("닉네임");
|
||||
expect(toolOptionNames[0]).toEqual(["search_tools"]);
|
||||
expect(toolOptionNames[1]).toEqual(["search_tools", "validate-nickname"]);
|
||||
});
|
||||
|
||||
it("preserves native tool call responses for follow-up turns", async () => {
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
|
||||
@@ -216,4 +293,291 @@ describe("runWithTools", () => {
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns permanent tool failures to the model instead of throwing", async () => {
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
seenMessages.push(messages);
|
||||
|
||||
if (seenMessages.length === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ value: string }, { ok: boolean }> = {
|
||||
name: "explode",
|
||||
description: "항상 실패",
|
||||
argsSchema: z.object({
|
||||
value: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
}),
|
||||
execute: () => Promise.reject(new Error("boom")),
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("기본 안내");
|
||||
const toolMessage = seenMessages[1]?.find(
|
||||
(message) => message.role === "tool",
|
||||
);
|
||||
expect(JSON.parse(String(toolMessage?.content))).toMatchObject({
|
||||
errorKind: "permanent",
|
||||
});
|
||||
});
|
||||
|
||||
it("allows one self-correction hop when tool arguments are invalid", async () => {
|
||||
let executeCount = 0;
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
seenMessages.push(messages);
|
||||
|
||||
if (seenMessages.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (seenMessages.length === 2) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "프리리님 기준으로 확인했어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "프리리님 기준으로 확인했어요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ name: string }, { ok: boolean }> = {
|
||||
name: "needs-name",
|
||||
description: "이름 필요",
|
||||
argsSchema: z.object({
|
||||
name: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
}),
|
||||
execute: () => {
|
||||
executeCount += 1;
|
||||
return Promise.resolve({ ok: true });
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("프리리님");
|
||||
expect(executeCount).toBe(1);
|
||||
expect(JSON.stringify(seenMessages[1])).toContain("schema");
|
||||
});
|
||||
|
||||
it("retries transient tool errors once before continuing", async () => {
|
||||
let executeCount = 0;
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
if (messages.some((message) => message.role === "tool")) {
|
||||
return Promise.resolve({
|
||||
content: "재시도 후 확인됐어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "재시도 후 확인됐어요.",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: '<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ value: string }, { ok: boolean }> = {
|
||||
name: "flaky",
|
||||
description: "일시 실패",
|
||||
argsSchema: z.object({
|
||||
value: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
}),
|
||||
execute: () => {
|
||||
executeCount += 1;
|
||||
if (executeCount === 1) {
|
||||
const error = new Error("timeout while calling dependency");
|
||||
error.name = "AbortError";
|
||||
return Promise.reject(error);
|
||||
}
|
||||
|
||||
return Promise.resolve({ ok: true });
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toBe("재시도 후 확인됐어요.");
|
||||
expect(executeCount).toBe(2);
|
||||
});
|
||||
|
||||
it("grants one extra hop for precondition failures", async () => {
|
||||
let executeCount = 0;
|
||||
const llmClientStub = {
|
||||
chatCompletion: () => {
|
||||
if (executeCount === 0) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (executeCount === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "추가 확인까지 마쳤어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "추가 확인까지 마쳤어요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<
|
||||
{ value: string },
|
||||
{ ok: boolean; errorKind?: string | undefined; message?: string | undefined }
|
||||
> = {
|
||||
name: "needs-session",
|
||||
description: "세션 필요",
|
||||
argsSchema: z.object({
|
||||
value: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
errorKind: z.string().optional(),
|
||||
message: z.string().optional(),
|
||||
}),
|
||||
execute: () => {
|
||||
executeCount += 1;
|
||||
if (executeCount === 1) {
|
||||
return Promise.resolve({
|
||||
ok: false,
|
||||
errorKind: "precondition-failed",
|
||||
message: "세션이 없어요.",
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({ ok: true });
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
maxHops: 1,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("추가 확인");
|
||||
expect(executeCount).toBe(2);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user