Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent a900016be4
commit 221e9391e0
71 files changed
+7057 -102

No files matched your search

@@ -3,6 +3,8 @@ import { z } from "zod";
import { runWithTools } from "./tool-loop.js";
import type { Tool } from "@tools/tool";
import { ToolRegistry } from "@tools/registry";
import { createToolSearchTool } from "@tools/tool-search";
describe("runWithTools", () => {
it("executes a tool call and returns the model's final answer", async () => {
@@ -106,6 +108,81 @@ describe("runWithTools", () => {
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
});
it("expands searched tools for the next tool-call hop", async () => {
const toolOptionNames: string[][] = [];
const llmClientStub = {
chatCompletion: (
_messages: Array<Record<string, unknown>>,
options?: { tools?: Array<{ name: string }> },
) => {
toolOptionNames.push(options?.tools?.map((tool) => tool.name) ?? []);
if (toolOptionNames.length === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
},
});
}
if (toolOptionNames.length === 2) {
return Promise.resolve({
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "닉네임 형식을 다시 맞춰주세요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "닉네임 형식을 다시 맞춰주세요.",
},
});
},
};
const validateTool: Tool<{ nickname: string }, { valid: boolean }> = {
argsSchema: z.object({
nickname: z.string(),
}),
description: "닉네임 형식을 검사한다.",
execute: () => Promise.resolve({ valid: false }),
name: "validate-nickname",
resultSchema: z.object({
valid: z.boolean(),
}),
};
const registry = new ToolRegistry().register(validateTool);
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "닉네임 확인해줘" }],
toolContext: { toolRegistry: registry },
tools: [createToolSearchTool(registry)],
});
expect(result).toContain("닉네임");
expect(toolOptionNames[0]).toEqual(["search_tools"]);
expect(toolOptionNames[1]).toEqual(["search_tools", "validate-nickname"]);
});
it("preserves native tool call responses for follow-up turns", async () => {
const seenMessages: Array<Array<Record<string, unknown>>> = [];
@@ -216,4 +293,291 @@ describe("runWithTools", () => {
),
).toBe(true);
});
it("returns permanent tool failures to the model instead of throwing", async () => {
const seenMessages: Array<Array<Record<string, unknown>>> = [];
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
seenMessages.push(messages);
if (seenMessages.length === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
},
});
},
};
const tool: Tool<{ value: string }, { ok: boolean }> = {
name: "explode",
description: "항상 실패",
argsSchema: z.object({
value: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
}),
execute: () => Promise.reject(new Error("boom")),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toContain("기본 안내");
const toolMessage = seenMessages[1]?.find(
(message) => message.role === "tool",
);
expect(JSON.parse(String(toolMessage?.content))).toMatchObject({
errorKind: "permanent",
});
});
it("allows one self-correction hop when tool arguments are invalid", async () => {
let executeCount = 0;
const seenMessages: Array<Array<Record<string, unknown>>> = [];
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
seenMessages.push(messages);
if (seenMessages.length === 1) {
return Promise.resolve({
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
},
});
}
if (seenMessages.length === 2) {
return Promise.resolve({
content:
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "프리리님 기준으로 확인했어요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "프리리님 기준으로 확인했어요.",
},
});
},
};
const tool: Tool<{ name: string }, { ok: boolean }> = {
name: "needs-name",
description: "이름 필요",
argsSchema: z.object({
name: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
}),
execute: () => {
executeCount += 1;
return Promise.resolve({ ok: true });
},
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toContain("프리리님");
expect(executeCount).toBe(1);
expect(JSON.stringify(seenMessages[1])).toContain("schema");
});
it("retries transient tool errors once before continuing", async () => {
let executeCount = 0;
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
if (messages.some((message) => message.role === "tool")) {
return Promise.resolve({
content: "재시도 후 확인됐어요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "재시도 후 확인됐어요.",
},
});
}
return Promise.resolve({
content: '<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
},
});
},
};
const tool: Tool<{ value: string }, { ok: boolean }> = {
name: "flaky",
description: "일시 실패",
argsSchema: z.object({
value: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
}),
execute: () => {
executeCount += 1;
if (executeCount === 1) {
const error = new Error("timeout while calling dependency");
error.name = "AbortError";
return Promise.reject(error);
}
return Promise.resolve({ ok: true });
},
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toBe("재시도 후 확인됐어요.");
expect(executeCount).toBe(2);
});
it("grants one extra hop for precondition failures", async () => {
let executeCount = 0;
const llmClientStub = {
chatCompletion: () => {
if (executeCount === 0) {
return Promise.resolve({
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
},
});
}
if (executeCount === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "추가 확인까지 마쳤어요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "추가 확인까지 마쳤어요.",
},
});
},
};
const tool: Tool<
{ value: string },
{ ok: boolean; errorKind?: string | undefined; message?: string | undefined }
> = {
name: "needs-session",
description: "세션 필요",
argsSchema: z.object({
value: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
errorKind: z.string().optional(),
message: z.string().optional(),
}),
execute: () => {
executeCount += 1;
if (executeCount === 1) {
return Promise.resolve({
ok: false,
errorKind: "precondition-failed",
message: "세션이 없어요.",
});
}
return Promise.resolve({ ok: true });
},
};
const result = await runWithTools({
llmClient: llmClientStub as never,
maxHops: 1,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toContain("추가 확인");
expect(executeCount).toBe(2);
});
});