Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent 6f09408423
commit 41dcfcdef7
71 files changed
+7057 -102

No files matched your search

@@ -0,0 +1,137 @@
import { describe, expect, it } from "vitest";
import type { AgentRequest } from "@agent/types";
import { decideEarlyAbstention } from "./abstention.js";
import type {
ClassifiedIntent,
DerivedState,
Intent,
} from "../state/onboarding-state.js";
import type { Blackboard } from "../types.js";
function request(overrides?: Partial<AgentRequest>): AgentRequest {
return {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "테스터",
trigger: "user-message",
userMessage: "뭐죠",
...overrides,
};
}
function classified(
intent: Intent["kind"],
confidence: number,
): ClassifiedIntent {
return {
intent: { kind: intent } as Intent,
confidence,
source: "llm",
};
}
function derived(intent: Intent["kind"]): DerivedState {
return {
phase: { kind: "fresh" },
intent: { kind: intent } as Intent,
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
};
}
function blackboard(overrides?: Partial<Blackboard>): Blackboard {
return {
flaggedIssues: [],
openQuestions: [],
recentNicknameAttempts: [],
toneMemory: {},
...overrides,
};
}
const thresholds = {
abstain: 0.35,
clarify: 0.45,
};
describe("decideEarlyAbstention", () => {
it("returns no-response for high-confidence off-topic intent", () => {
const decision = decideEarlyAbstention({
blackboard: null,
classified: classified("off-topic-to-other", 0.95),
derived: derived("off-topic-to-other"),
request: request(),
thresholds,
});
expect(decision.action.kind).toBe("no-response");
expect(decision.reason).toBe("off-topic-strong");
});
it("asks for clarification on low-confidence early unknown turns", () => {
const decision = decideEarlyAbstention({
blackboard: blackboard(),
classified: classified("unknown", 0.4),
derived: derived("unknown"),
request: request(),
thresholds,
});
expect(decision.action).toEqual({
kind: "clarify",
templateKey: "abstain-clarify-default",
});
expect(decision.reason).toBe("low-confidence-early-turn");
});
it("returns no-response for repeated low-confidence intent loops", () => {
const decision = decideEarlyAbstention({
blackboard: blackboard({
lastIntent: "unknown",
sameIntentStreak: 2,
}),
classified: classified("unknown", 0.2),
derived: derived("unknown"),
request: request({ context: { turnIndex: 3 } }),
thresholds,
});
expect(decision.action.kind).toBe("no-response");
expect(decision.reason).toBe("low-confidence-repeated");
expect(decision.features.sameIntentStreak).toBe(3);
});
it("prioritizes flagged confusion when low confidence and issues are present", () => {
const decision = decideEarlyAbstention({
blackboard: blackboard({
flaggedIssues: ["닉네임 형식 반복 오류"],
}),
classified: classified("unknown", 0.3),
derived: derived("unknown"),
request: request({ context: { turnIndex: 2 } }),
thresholds,
});
expect(decision.action.kind).toBe("no-response");
expect(decision.reason).toBe("flagged-confused");
});
it("proceeds on clarify threshold boundary", () => {
const decision = decideEarlyAbstention({
blackboard: blackboard(),
classified: classified("unknown", thresholds.clarify),
derived: derived("unknown"),
request: request(),
thresholds,
});
expect(decision.action.kind).toBe("proceed");
expect(decision.reason).toBeNull();
});
});
+158
View File
@@ -0,0 +1,158 @@
import type { AgentRequest } from "@agent/types";
import type {
ClassifiedIntent,
DerivedState,
Intent,
} from "../state/onboarding-state.js";
import type { Blackboard } from "../types.js";
export type AbstentionAction =
| { kind: "proceed" }
| { kind: "clarify"; templateKey: string }
| { kind: "no-response" };
export interface AbstentionDecision {
action: AbstentionAction;
reason:
| "low-confidence-early-turn"
| "low-confidence-repeated"
| "off-topic-strong"
| "flagged-confused"
| null;
score: number;
features: {
classifyConfidence: number;
intentKind: Intent["kind"];
turnIndex: number;
sameIntentStreak: number;
flaggedIssues: number;
};
}
function clampRatio(value: number): number {
return Math.max(0, Math.min(1, value));
}
function roundRatio(value: number): number {
return Number(clampRatio(value).toFixed(3));
}
function getExplicitTurnIndex(request: AgentRequest): number | null {
const value = request.context?.["turnIndex"];
return typeof value === "number" && Number.isFinite(value) && value >= 0
? Math.floor(value)
: null;
}
function inferTurnIndex(request: AgentRequest, blackboard: Blackboard | null): number {
const explicit = getExplicitTurnIndex(request);
if (explicit !== null) {
return explicit;
}
if (!blackboard?.lastIntent) {
return 0;
}
return blackboard.toneMemory.greetedAt ? 2 : 1;
}
function getSameIntentStreak(
intentKind: Intent["kind"],
blackboard: Blackboard | null,
): number {
if (!blackboard || blackboard.lastIntent !== intentKind) {
return 1;
}
return Math.max(2, (blackboard.sameIntentStreak ?? 1) + 1);
}
function computeScore(input: {
confidence: number;
flaggedIssues: number;
intentKind: Intent["kind"];
sameIntentStreak: number;
turnIndex: number;
}): number {
const lowConfidence = 1 - input.confidence;
const offTopicBoost = input.intentKind === "off-topic-to-other" ? 0.25 : 0;
const flaggedBoost = input.flaggedIssues > 0 ? 0.2 : 0;
const repeatedBoost = input.sameIntentStreak >= 3 ? 0.15 : 0;
const earlyBoost = input.turnIndex <= 1 ? 0.05 : 0;
return roundRatio(
lowConfidence + offTopicBoost + flaggedBoost + repeatedBoost + earlyBoost,
);
}
function makeDecision(
features: AbstentionDecision["features"],
action: AbstentionAction,
reason: AbstentionDecision["reason"],
): AbstentionDecision {
return {
action,
reason,
score: computeScore({
confidence: features.classifyConfidence,
flaggedIssues: features.flaggedIssues,
intentKind: features.intentKind,
sameIntentStreak: features.sameIntentStreak,
turnIndex: features.turnIndex,
}),
features,
};
}
export function decideEarlyAbstention(input: {
classified: ClassifiedIntent;
derived: DerivedState;
blackboard: Blackboard | null;
request: AgentRequest;
thresholds: {
abstain: number;
clarify: number;
};
}): AbstentionDecision {
const confidence = clampRatio(input.classified.confidence);
const intentKind = input.classified.intent.kind;
const features = {
classifyConfidence: confidence,
intentKind,
turnIndex: inferTurnIndex(input.request, input.blackboard),
sameIntentStreak: getSameIntentStreak(intentKind, input.blackboard),
flaggedIssues: input.blackboard?.flaggedIssues.length ?? 0,
};
if (intentKind === "off-topic-to-other" && confidence > 0.9) {
return makeDecision(features, { kind: "no-response" }, "off-topic-strong");
}
if (
(features.flaggedIssues > 0 || features.sameIntentStreak >= 3) &&
confidence < input.thresholds.abstain
) {
return makeDecision(
features,
{ kind: "no-response" },
features.flaggedIssues > 0
? "flagged-confused"
: "low-confidence-repeated",
);
}
if (
features.turnIndex <= 1 &&
confidence < input.thresholds.clarify &&
(intentKind === "unknown" || intentKind === "chitchat")
) {
return makeDecision(
features,
{ kind: "clarify", templateKey: "abstain-clarify-default" },
"low-confidence-early-turn",
);
}
return makeDecision(features, { kind: "proceed" }, null);
}
@@ -0,0 +1,111 @@
import { afterEach, describe, expect, it } from "vitest";
import type { ChatMessage } from "@llm/types";
import type { DerivedState } from "../state/onboarding-state.js";
import { OnboardingCritic } from "./critic.js";
function derived(overrides?: Partial<DerivedState>): DerivedState {
return {
allowShortResponse: false,
intent: { kind: "help-how" },
phase: { kind: "photo-pending" },
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 256,
tier: 2,
...overrides,
};
}
const messages: ChatMessage[] = [
{
role: "user",
content: "하트 인증은 어떻게 해요?",
},
];
describe("OnboardingCritic", () => {
afterEach(() => {
delete process.env["AGENT_CRITIC_MAX_ITERATIONS"];
});
it("iteratively rewrites and then keeps the improved draft", async () => {
const calls: Array<{ content: string; options: Record<string, unknown> }> = [];
const llmClient = {
chatCompletion: (
nextMessages: ChatMessage[],
options: Record<string, unknown>,
) => {
calls.push({
content: nextMessages.at(-1)?.content ?? "",
options,
});
return Promise.resolve({
content:
calls.length === 1
? JSON.stringify({
decision: "rewrite",
rationale: "too broad",
rubricScores: { accuracy: 0.6, scope: 0.4, tone: 0.8 },
text: "하트 인증 화면 원본 스크린샷만 올려주시면 돼요.",
})
: JSON.stringify({
decision: "keep",
rationale: "good",
rubricScores: { accuracy: 0.9, scope: 0.9, tone: 0.8 },
}),
endpoint: "mock://llm",
message: { role: "assistant", content: "{}" },
provider: "llama",
raw: {},
});
},
};
const critic = new OnboardingCritic(llmClient as never);
const result = await critic.critiqueIterative(
"여러 절차를 모두 진행하세요.",
derived(),
messages,
{ maxIterations: 2 },
);
expect(result).toMatchObject({
abstained: false,
changed: true,
iterations: 2,
text: "하트 인증 화면 원본 스크린샷만 올려주시면 돼요.",
});
expect(result.rubricScores.accuracy).toBe(0.9);
expect(calls[0]?.options).toMatchObject({ jsonMode: true, role: "critic" });
});
it("can abstain from unsafe drafts", async () => {
const llmClient = {
chatCompletion: () =>
Promise.resolve({
content: JSON.stringify({
decision: "abstain",
rationale: "off topic",
rubricScores: { accuracy: 0.1, scope: 0.1, tone: 0.5 },
text: "[NO_RESPONSE]",
}),
endpoint: "mock://llm",
message: { role: "assistant", content: "{}" },
provider: "llama",
raw: {},
}),
};
const critic = new OnboardingCritic(llmClient as never);
const result = await critic.critiqueIterative(
"상관없는 답변",
derived(),
messages,
{ maxIterations: 2 },
);
expect(result.abstained).toBe(true);
expect(result.text).toBe("[NO_RESPONSE]");
});
});
+192
View File
@@ -1,5 +1,6 @@
import type { ChatMessage } from "@llm/types";
import type { LlmClient } from "@llm/llm-client";
import { z } from "zod";
import {
getCriticMaxTokens,
getLlmTemperature,
@@ -13,6 +14,29 @@ export interface CritiqueResult {
text: string;
}
export interface CritiqueResultV2 extends CritiqueResult {
abstained: boolean;
iterations: number;
prmScoreAfter?: number | undefined;
prmScoreBefore?: number | undefined;
rubricScores: {
accuracy: number;
scope: number;
tone: number;
};
}
const iterativeCriticSchema = z.object({
decision: z.enum(["keep", "rewrite", "abstain"]),
rationale: z.string().trim().max(300).optional(),
rubricScores: z.object({
accuracy: z.number().min(0).max(1),
scope: z.number().min(0).max(1),
tone: z.number().min(0).max(1),
}),
text: z.string().trim().max(1_000).optional(),
});
function serializeMessages(messages: ChatMessage[]): string {
return messages
.map((message) => `[${message.role}] ${message.content}`)
@@ -47,6 +71,67 @@ function buildCriticMessages(
];
}
function buildIterativeCriticMessages(
draft: string,
derived: DerivedState,
messages: ChatMessage[],
iteration: number,
): ChatMessage[] {
return [
{
role: "system",
content: [
"당신은 온보딩 응답을 반복 검토하는 critic입니다.",
"반드시 JSON object만 출력하세요.",
"decision은 keep, rewrite, abstain 중 하나입니다.",
"rewrite이면 text에 유저에게 바로 보낼 최종 한국어 답변만 넣으세요.",
"abstain이면 text는 [NO_RESPONSE]로 두세요.",
"rubricScores는 accuracy, scope, tone을 0..1 사이 숫자로 채우세요.",
].join("\n"),
},
{
role: "user",
content: [
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}`,
`[ITERATION] ${iteration}`,
"[CONVERSATION]",
serializeMessages(messages),
"[DRAFT]",
draft,
"",
'JSON schema: {"decision":"keep|rewrite|abstain","text":"...","rubricScores":{"accuracy":0.8,"scope":0.8,"tone":0.8},"rationale":"..."}',
].join("\n\n"),
},
];
}
function scoreRubric(scores: CritiqueResultV2["rubricScores"]): number {
return scores.accuracy + scores.scope + scores.tone;
}
function emptyRubric(): CritiqueResultV2["rubricScores"] {
return {
accuracy: 0,
scope: 0,
tone: 0,
};
}
function parseIterativeCriticResponse(
content: string | undefined,
): z.infer<typeof iterativeCriticSchema> | null {
if (!content?.trim()) {
return null;
}
try {
const parsed = iterativeCriticSchema.safeParse(JSON.parse(content));
return parsed.success ? parsed.data : null;
} catch {
return null;
}
}
export class OnboardingCritic {
constructor(private readonly llmClient: LlmClient) {}
@@ -90,4 +175,111 @@ export class OnboardingCritic {
text,
};
}
async critiqueIterative(
draft: string,
derived: DerivedState,
messages: ChatMessage[],
options: {
deadlineMs?: number | undefined;
maxIterations: number;
},
): Promise<CritiqueResultV2> {
const traceStep = getActiveProcessTrace()?.startStep("critic");
const maxIterations = Math.max(1, options.maxIterations);
let current = draft;
let changed = false;
let bestRubric = emptyRubric();
let bestScore = -1;
let rationale = "max iterations reached";
for (let iteration = 1; iteration <= maxIterations; iteration++) {
if (options.deadlineMs !== undefined && options.deadlineMs <= 0) {
rationale = "deadline exceeded";
break;
}
const response = await this.llmClient.chatCompletion(
buildIterativeCriticMessages(current, derived, messages, iteration),
{
deadlineMs: options.deadlineMs,
jsonMode: true,
maxTokens: getCriticMaxTokens(),
modelOverride: getTier2CriticModel(),
role: "critic",
temperature: Math.max(0, getLlmTemperature() - 0.15),
tier: 2,
},
);
const parsed = parseIterativeCriticResponse(response?.content);
if (!parsed) {
rationale = "invalid critic response";
break;
}
const nextScore = scoreRubric(parsed.rubricScores);
if (nextScore >= bestScore) {
bestRubric = parsed.rubricScores;
bestScore = nextScore;
}
rationale = parsed.rationale ?? parsed.decision;
if (parsed.decision === "abstain") {
traceStep?.finish({
abstained: true,
decision: "rewrite",
iterations: iteration,
rationale,
rubricScores: bestRubric,
});
return {
abstained: true,
changed: true,
iterations: iteration,
rubricScores: bestRubric,
text: "[NO_RESPONSE]",
};
}
if (parsed.decision === "keep") {
traceStep?.finish({
abstained: false,
decision: changed ? "rewrite" : "keep",
iterations: iteration,
rationale,
rubricScores: bestRubric,
});
return {
abstained: false,
changed,
iterations: iteration,
rubricScores: bestRubric,
text: current,
};
}
const nextText = parsed.text?.trim();
if (!nextText || nextText === current || nextScore < bestScore) {
break;
}
current = nextText;
changed = true;
}
traceStep?.finish({
abstained: false,
decision: changed ? "rewrite" : "keep",
iterations: maxIterations,
rationale,
rubricScores: bestRubric,
});
return {
abstained: false,
changed,
iterations: maxIterations,
rubricScores: bestRubric,
text: current,
};
}
}
@@ -1,7 +1,9 @@
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it } from "vitest";
import { z } from "zod";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { OnboardingOrchestrator } from "./orchestrator.js";
import type { DerivedState } from "../state/onboarding-state.js";
import type { Tool } from "@tools/tool";
function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
return {
@@ -17,6 +19,11 @@ function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
}
describe("OnboardingOrchestrator", () => {
afterEach(() => {
delete process.env["AGENT_PROGRAMMATIC_TOOL_CALLING"];
delete process.env["AGENT_SKILL_MD_MODE"];
});
it("uses direct response path for tier 0", async () => {
const calls: Array<{
messages: ChatMessage[];
@@ -107,6 +114,165 @@ describe("OnboardingOrchestrator", () => {
expect(calls[1]?.messages[0]?.content).toContain("[PLAN]");
});
it("runs a planner toolPlan before the responder when enabled", async () => {
process.env["AGENT_PROGRAMMATIC_TOOL_CALLING"] = "true";
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
if (calls.length === 1) {
return Promise.resolve({
content: JSON.stringify({
approach: "validate-then-explain",
targetPhase: "nickname-rejected",
mustInclude: ["검사 결과"],
mustAvoid: [],
suggestedTools: ["validate-nickname"],
toolPlan: {
composeWith: "all-then-llm",
steps: [
{
id: "validate",
tool: "validate-nickname",
argsFromContext: {
nickname: "$request.nickname",
},
},
],
},
tone: "friendly",
targetLengthChars: 120,
rationale: "도구 선실행",
}),
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "{}" } }] },
message: { role: "assistant" as const, content: "{}" },
});
}
return Promise.resolve({
content: "검사 결과 형식이 맞아요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "검사 결과 형식이 맞아요." } }] },
message: { role: "assistant" as const, content: "검사 결과 형식이 맞아요." },
});
},
};
const tool: Tool<{ nickname: string }, { valid: boolean }> = {
name: "validate-nickname",
description: "닉네임 검사",
argsSchema: z.object({
nickname: z.string(),
}),
resultSchema: z.object({
valid: z.boolean(),
}),
execute: (args) => Promise.resolve({ valid: args.nickname.includes("/") }),
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ responseMode: "llm-with-tools", tier: 1 }),
[
{ role: "system", content: "기본 시스템 프롬프트" },
{ role: "user", content: "닉네임 봐줘" },
],
{
request: {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
},
toolContext: {},
tools: [tool],
},
);
expect(result).toBe("검사 결과 형식이 맞아요.");
expect(calls).toHaveLength(2);
expect(calls[1]?.messages[0]?.content).toContain("[TOOL_PLAN_RESULTS]");
expect(calls[1]?.messages[0]?.content).toContain('"valid":true');
expect(calls[1]?.options?.tools).toBeUndefined();
});
it("injects requested SKILL.md resources before responder when enabled", async () => {
process.env["AGENT_SKILL_MD_MODE"] = "true";
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
if (calls.length === 1) {
return Promise.resolve({
content: JSON.stringify({
approach: "answer-direct",
targetPhase: "photo-pending",
mustInclude: ["하트"],
mustAvoid: [],
suggestedTools: [],
requestResources: ["heart-rule"],
tone: "friendly",
targetLengthChars: 120,
rationale: "리소스 참고",
}),
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "{}" } }] },
message: { role: "assistant" as const, content: "{}" },
});
}
return Promise.resolve({
content: "커버 보기에서 하트를 빨간색으로 바꿔주세요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: {
choices: [
{ message: { content: "커버 보기에서 하트를 빨간색으로 바꿔주세요." } },
],
},
message: {
role: "assistant" as const,
content: "커버 보기에서 하트를 빨간색으로 바꿔주세요.",
},
});
},
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ tier: 1 }),
[
{ role: "system", content: "기본 시스템 프롬프트" },
{ role: "user", content: "하트 어디서 해요?" },
],
{ toolContext: {} },
);
expect(result).toContain("커버 보기");
expect(calls[1]?.messages[0]?.content).toContain("[RESOURCE:heart-rule]");
});
it("runs critic after responder for tier 2 and applies rewrite", async () => {
const calls: Array<{
messages: ChatMessage[];
+129 -7
View File
@@ -1,8 +1,11 @@
import { getContext, getRequestLogger } from "@obs/context";
import type { AgentRequest } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { Tool, ToolContext } from "@tools/tool";
import {
getAgentCriticMaxIterations,
getAgentProgrammaticToolCalling,
getLlmTemperature,
getPlannerMaxTokens,
getTier0EstimatedCallMs,
@@ -16,14 +19,17 @@ import {
getTier2MaxTokens,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
import { buildSkillResourceAddendum } from "../prompts/builder.js";
import { OnboardingCritic } from "./critic.js";
import type { PlannerOutput } from "./planner-schema.js";
import { PlannerOutputSchema } from "./planner-schema.js";
import { getActiveProcessTrace } from "./trace.js";
import { runToolPlan, type ToolPlanResult } from "./tool-plan-runner.js";
import { runWithTools } from "./tool-loop.js";
export interface OrchestratorRunOptions {
maxTokens?: number | undefined;
request?: AgentRequest | undefined;
temperature?: number | undefined;
toolContext: ToolContext;
tools?: readonly Tool<unknown, unknown>[] | undefined;
@@ -100,6 +106,7 @@ function getDefaultPlan(derived: DerivedState): PlannerOutput {
targetPhase: derived.phase.kind,
mustInclude: [],
mustAvoid: [],
requestResources: [],
suggestedTools:
derived.responseMode === "llm-with-tools" ? ["validate-nickname"] : [],
tone: derived.allowShortResponse ? "concise" : "friendly",
@@ -154,6 +161,22 @@ function buildPlannerMessages(
suggestedTools: [
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context | verify-discord-nickname",
],
requestResources: [
"nickname-format | heart-rule | discord-format | age-policy",
],
toolPlan: {
composeWith: "all-then-llm",
steps: [
{
id: "step-id",
tool: "validate-nickname",
argsLiteral: { nickname: "value" },
argsFromContext: { chatroomId: "$request.chatroomId" },
dependsOn: [],
on: { error: "fallback-llm" },
},
],
},
tone: "friendly | concise | encouraging | neutral | apologetic",
targetLengthChars: 150,
rationale: "short reason",
@@ -184,6 +207,23 @@ function buildResponderAddendum(plan: PlannerOutput): string {
return lines.join("\n");
}
function buildToolPlanAddendum(result: ToolPlanResult): string {
return [
"[TOOL_PLAN_RESULTS]",
JSON.stringify({
durationMs: result.durationMs,
failures: result.failures.map((failure) => ({
kind: failure.kind,
message: failure.message,
retryable: failure.retryable,
toolName: failure.toolName,
})),
results: result.results,
}),
"위 toolPlan 결과를 사실 정보로 사용하고, 실패한 도구 결과는 추측으로 메우지 마세요.",
].join("\n");
}
function pickToolsForPlan(
plan: PlannerOutput,
tools: readonly Tool<unknown, unknown>[],
@@ -237,6 +277,7 @@ export class OnboardingOrchestrator {
messages,
modelOverride,
role: "responder",
routingDecision: derived.routingDecision,
tier: derived.tier,
temperature: options.temperature ?? getLlmTemperature(),
toolContext: options.toolContext,
@@ -249,6 +290,7 @@ export class OnboardingOrchestrator {
maxTokens,
modelOverride,
role: "responder",
routingDecision: derived.routingDecision,
temperature: options.temperature ?? getLlmTemperature(),
tier: derived.tier,
});
@@ -267,8 +309,10 @@ export class OnboardingOrchestrator {
approach: plan.approach,
mustAvoid: plan.mustAvoid,
mustInclude: plan.mustInclude,
requestResources: plan.requestResources,
suggestedTools: plan.suggestedTools,
targetLengthChars: plan.targetLengthChars,
toolPlanStepCount: plan.toolPlan?.steps.length,
});
return plan;
};
@@ -279,6 +323,7 @@ export class OnboardingOrchestrator {
maxTokens: getPlannerMaxTokens(),
modelOverride: getTier1PlannerModel(),
role: "planner",
routingDecision: derived.routingDecision,
temperature: 0,
tier: 1,
});
@@ -313,6 +358,7 @@ export class OnboardingOrchestrator {
maxTokens: getPlannerMaxTokens(),
modelOverride: getTier1PlannerModel(),
role: "planner",
routingDecision: derived.routingDecision,
temperature: 0,
tier: 1,
});
@@ -372,11 +418,32 @@ export class OnboardingOrchestrator {
buildResponderAddendum(plan),
);
const forceTools = derived.responseMode === "llm-with-tools";
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
let finalResponderMessages = responderMessages;
let responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
const resourceAddendum = buildSkillResourceAddendum(plan.requestResources);
if (resourceAddendum) {
finalResponderMessages = injectSystemAddendum(
finalResponderMessages,
resourceAddendum,
);
}
const toolPlanResult = await this.runProgrammaticToolPlan(
plan,
finalResponderMessages,
options,
);
if (toolPlanResult && !toolPlanResult.fallbackToLlm) {
finalResponderMessages = injectSystemAddendum(
finalResponderMessages,
buildToolPlanAddendum(toolPlanResult),
);
responderTools = [];
}
return this.runDirect(
derived,
responderMessages,
finalResponderMessages,
options,
responderTools,
options.maxTokens ?? getTier1MaxTokens(),
@@ -395,11 +462,32 @@ export class OnboardingOrchestrator {
buildResponderAddendum(plan),
);
const forceTools = derived.responseMode === "llm-with-tools";
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
let finalResponderMessages = responderMessages;
let responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
const resourceAddendum = buildSkillResourceAddendum(plan.requestResources);
if (resourceAddendum) {
finalResponderMessages = injectSystemAddendum(
finalResponderMessages,
resourceAddendum,
);
}
const toolPlanResult = await this.runProgrammaticToolPlan(
plan,
finalResponderMessages,
options,
);
if (toolPlanResult && !toolPlanResult.fallbackToLlm) {
finalResponderMessages = injectSystemAddendum(
finalResponderMessages,
buildToolPlanAddendum(toolPlanResult),
);
responderTools = [];
}
const draft = await this.runDirect(
derived,
responderMessages,
finalResponderMessages,
options,
responderTools,
options.maxTokens ?? getTier2MaxTokens(),
@@ -410,12 +498,46 @@ export class OnboardingOrchestrator {
return null;
}
const critique = await this.critic.critique(draft, derived, responderMessages, {
deadlineMs: getRemainingDeadlineMs(),
});
const maxIterations = getAgentCriticMaxIterations();
const critique =
maxIterations > 1
? await this.critic.critiqueIterative(
draft,
derived,
finalResponderMessages,
{
deadlineMs: getRemainingDeadlineMs(),
maxIterations,
},
)
: await this.critic.critique(draft, derived, finalResponderMessages, {
deadlineMs: getRemainingDeadlineMs(),
});
return critique.text;
}
private async runProgrammaticToolPlan(
plan: PlannerOutput,
_messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<ToolPlanResult | null> {
if (
!getAgentProgrammaticToolCalling() ||
!plan.toolPlan ||
!options.request ||
!options.tools?.length
) {
return null;
}
return runToolPlan(
plan.toolPlan,
options.tools,
options.toolContext,
options.request,
);
}
async run(
derived: DerivedState,
messages: ChatMessage[],
@@ -38,15 +38,47 @@ export const PlannerToneSchema = z.enum([
"apologetic",
]);
export const ToolPlanStepSchema = z.object({
id: z.string().trim().min(1).max(40),
tool: PlannerSuggestedToolSchema,
argsFromContext: z.record(z.string(), z.string()).optional(),
argsLiteral: z.record(z.string(), z.unknown()).optional(),
dependsOn: z.array(z.string().trim().min(1)).max(3).optional(),
on: z
.object({
error: z.enum(["abort", "fallback-llm"]).optional(),
})
.optional(),
});
export const ToolPlanSchema = z.object({
steps: z.array(ToolPlanStepSchema).max(4),
composeWith: z.enum(["all-then-llm", "first-then-llm"]),
});
export const PlannerOutputSchema = z.object({
approach: PlannerApproachSchema,
targetPhase: PlannerTargetPhaseSchema.optional(),
mustInclude: z.array(z.string().min(1).max(80)).max(5).default([]),
mustAvoid: z.array(z.string().min(1).max(80)).max(5).default([]),
suggestedTools: z.array(PlannerSuggestedToolSchema).max(3).default([]),
requestResources: z
.array(
z.enum([
"nickname-format",
"heart-rule",
"discord-format",
"age-policy",
]),
)
.max(4)
.default([]),
toolPlan: ToolPlanSchema.optional(),
tone: PlannerToneSchema.default("friendly"),
targetLengthChars: z.number().int().min(20).max(500).default(150),
rationale: z.string().max(200),
});
export type PlannerOutput = z.infer<typeof PlannerOutputSchema>;
export type ToolPlan = z.infer<typeof ToolPlanSchema>;
export type ToolPlanStep = z.infer<typeof ToolPlanStepSchema>;
@@ -0,0 +1,394 @@
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import type { AgentRequest } from "@agent/types";
import type {
ClassifiedIntent,
DerivedState,
Intent,
OnboardingPhase,
} from "../state/onboarding-state.js";
import type { OnboardingSession } from "../types.js";
import {
decideHybridRouting,
decidePreGenerationRouting,
extractRouterFeatures,
loadLearnedRouterModel,
predictRoutingDecisionKind,
saveLearnedRouterModel,
trainLearnedRouterModel,
} from "./router.js";
function intent(kind: Intent["kind"]): Intent {
return { kind };
}
function request(overrides?: Partial<AgentRequest>): AgentRequest {
return {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "도와주세요",
...overrides,
};
}
function session(overrides?: Partial<OnboardingSession>): OnboardingSession {
return {
chatroomId: "room-1",
deadlineAt: "2026-05-02T00:00:00.000Z",
joinedAt: "2026-05-01T00:00:00.000Z",
kakaotalkNickname: "프리리/95/여/D",
kakaotalkNicknameAliases: [],
kakaotalkNicknameVerified: false,
lastEventAt: "2026-05-01T00:00:00.000Z",
nicknameVerified: false,
photoVerified: false,
status: "pending",
userId: "user-1",
...overrides,
};
}
function derived(
phase: OnboardingPhase,
nextIntent: Intent,
overrides?: Partial<DerivedState>,
): DerivedState {
return {
allowShortResponse: false,
intent: nextIntent,
phase,
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 256,
tier: 0,
...overrides,
};
}
function classified(
kind: Intent["kind"],
confidence: number,
): ClassifiedIntent {
return {
confidence,
intent: intent(kind),
source: "regex",
};
}
describe("decidePreGenerationRouting", () => {
it.each([
{
confidence: 0.91,
expected: { kind: "abstain", reason: "off-topic-strong" },
intentKind: "off-topic-to-other" as const,
name: "abstains on high-confidence off-topic",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.9,
expected: { kind: "tier0-only" },
intentKind: "off-topic-to-other" as const,
name: "does not abstain at the off-topic confidence boundary",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.3,
expected: { kind: "tier0-only" },
intentKind: "off-topic-to-other" as const,
name: "keeps low-confidence off-topic on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.96,
expected: { kind: "template-only", templateKey: "greeting" },
intentKind: "greeting" as const,
name: "uses a greeting template",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.95,
expected: { kind: "tier0-only" },
intentKind: "greeting" as const,
name: "does not template greeting at the confidence boundary",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.99,
expected: { kind: "template-only", templateKey: "greeting" },
intentKind: "greeting" as const,
name: "uses greeting template from a pending phase",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.91,
expected: { kind: "template-only", templateKey: "acknowledgement" },
intentKind: "acknowledgement" as const,
name: "uses an acknowledgement template",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.9,
expected: { kind: "tier0-only" },
intentKind: "acknowledgement" as const,
name: "does not template acknowledgement at the confidence boundary",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "template-only", templateKey: "completed" },
intentKind: "help-how" as const,
name: "keeps completed sessions on template path",
phase: { kind: "completed" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "status-question" as const,
name: "lets completed status questions through",
phase: { kind: "completed" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "template-only", templateKey: "completed" },
intentKind: "age-question" as const,
name: "uses completed template for completed phase simple questions",
phase: { kind: "completed" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "nickname-question" as const,
name: "routes nickname help through tools",
phase: { kind: "nickname-pending", current: "프리리" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "nickname-question" as const,
name: "routes nickname-rejected phase through tools",
phase: {
kind: "nickname-rejected",
current: "프리리/95/여D",
issues: [],
} as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "photo-question" as const,
name: "routes photo help through tools",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "photo-question" as const,
name: "routes photo help through tools from both-pending",
phase: { kind: "both-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "help-how" as const,
name: "routes general step-one help through tools",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "help-where" as const,
name: "routes where-help through tools",
phase: { kind: "both-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "help-how" as const,
name: "keeps later phase general help on tier0",
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "discord-verify-question" as const,
name: "routes discord verification through tools",
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-with-tools" },
intentKind: "discord-verify-question" as const,
name: "routes discord verification through tools from fresh phase",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-direct", skipPlanner: false },
intentKind: "status-question" as const,
name: "keeps deadline-near on tier1",
phase: { kind: "deadline-near", remaining: ["photo"] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier1-direct", skipPlanner: false },
intentKind: "age-question" as const,
name: "routes deadline-near simple questions through tier1",
phase: { kind: "deadline-near", remaining: ["nickname"] } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "age-question" as const,
name: "defaults simple questions to tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "age-question" as const,
name: "keeps age questions on tier0 during photo-pending",
phase: { kind: "photo-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "status-question" as const,
name: "keeps ordinary status questions on tier0",
phase: { kind: "both-pending" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "discord-question" as const,
name: "keeps generic discord questions on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "unknown" as const,
name: "keeps unknown intent on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "chitchat" as const,
name: "keeps chitchat on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "age-question" as const,
name: "keeps simple questions on tier0 during photo rejection",
phase: { kind: "photo-rejected", reason: "unclear" } as OnboardingPhase,
},
{
confidence: 0.8,
expected: { kind: "tier0-only" },
intentKind: "nickname-question" as const,
name: "keeps nickname question on tier0 after step one",
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
},
{
confidence: 0.89,
expected: { kind: "tier0-only" },
intentKind: "acknowledgement" as const,
name: "keeps lower-confidence acknowledgement on tier0",
phase: { kind: "fresh" } as OnboardingPhase,
},
])("$name", ({ confidence, expected, intentKind, phase }) => {
const nextIntent = intent(intentKind);
expect(
decidePreGenerationRouting({
classified: classified(intentKind, confidence),
derived: derived(phase, nextIntent),
request: request(),
session: session(),
}),
).toEqual(expected);
});
it("trains, saves, loads, and applies a learned router model", () => {
const lowConfidenceInput = {
classified: classified("age-question", 0.25),
derived: derived({ kind: "fresh" }, intent("age-question")),
request: request({ userMessage: "나이 제한이 있나요?" }),
session: session(),
};
const highConfidenceInput = {
classified: classified("age-question", 0.95),
derived: derived({ kind: "fresh" }, intent("age-question")),
request: request({ userMessage: "나이 제한이 있나요?" }),
session: session(),
};
const model = trainLearnedRouterModel(
[
{
features: extractRouterFeatures(lowConfidenceInput),
label: "tier1-direct",
},
{
features: extractRouterFeatures(highConfidenceInput),
label: "tier0-only",
},
],
{
epochs: 200,
learningRate: 0.5,
trainedAt: "2026-05-01T00:00:00.000Z",
},
);
const path = join(mkdtempSync(join(tmpdir(), "router-")), "weights.json");
saveLearnedRouterModel(path, model);
const loaded = loadLearnedRouterModel(path);
expect(loaded?.trainedAt).toBe("2026-05-01T00:00:00.000Z");
expect(
predictRoutingDecisionKind(model, extractRouterFeatures(lowConfidenceInput)),
).toBe("tier1-direct");
expect(
predictRoutingDecisionKind(model, extractRouterFeatures(highConfidenceInput)),
).toBe("tier0-only");
expect(
decideHybridRouting(lowConfidenceInput, { learnedModel: model }),
).toEqual({
kind: "tier1-direct",
skipPlanner: false,
});
});
it("keeps static abstain and template guards in hybrid routing", () => {
const offTopicInput = {
classified: classified("off-topic-to-other", 0.99),
derived: derived({ kind: "fresh" }, intent("off-topic-to-other")),
request: request({ userMessage: "다른 주제로 얘기하자" }),
session: session(),
};
const model = trainLearnedRouterModel(
[
{
features: extractRouterFeatures(offTopicInput),
label: "tier2-full",
},
],
{ epochs: 10 },
);
expect(decideHybridRouting(offTopicInput, { learnedModel: model })).toEqual({
kind: "abstain",
reason: "off-topic-strong",
});
});
});
+379
View File
@@ -0,0 +1,379 @@
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import type { AgentRequest } from "@agent/types";
import type {
ClassifiedIntent,
DerivedState,
Intent,
OnboardingPhase,
} from "../state/onboarding-state.js";
import type { OnboardingSession } from "../types.js";
export type RoutingDecision =
| { kind: "abstain"; reason: string }
| { kind: "template-only"; templateKey: string }
| { kind: "tier0-only" }
| { kind: "tier1-direct"; skipPlanner: boolean }
| { kind: "tier1-with-tools" }
| { kind: "tier2-full" };
export const ROUTING_DECISION_KINDS = [
"abstain",
"template-only",
"tier0-only",
"tier1-direct",
"tier1-with-tools",
"tier2-full",
] as const;
export type RoutingDecisionKind = (typeof ROUTING_DECISION_KINDS)[number];
export interface RouterFeatureVector {
classifyConfidence: number;
classifySource: 0 | 1;
intentKind: string;
messageLength: number;
phaseKind: string;
responseModeTools: 0 | 1;
sessionCompleted: 0 | 1;
tier: 0 | 1 | 2;
trigger: string;
}
export interface LearnedRouterModel {
decisionKinds: RoutingDecisionKind[];
featureNames: string[];
trainedAt: string;
version: 1;
weights: number[][];
}
export interface RouterTrainingSample {
features: RouterFeatureVector;
label: RoutingDecisionKind;
}
export interface RouterInput {
classified: ClassifiedIntent;
derived: DerivedState;
request: AgentRequest;
session: OnboardingSession | null;
}
export const ROUTER_FEATURE_NAMES = [
"bias",
"classifyConfidence",
"classifySource",
"intentHash",
"phaseHash",
"triggerHash",
"messageLength",
"tier",
"responseModeTools",
"sessionCompleted",
] as const;
const modelCache = new Map<string, LearnedRouterModel | null>();
function round(value: number): number {
return Number(value.toFixed(6));
}
function clamp01(value: number): number {
return Math.max(0, Math.min(1, value));
}
function stableBucket(value: string): number {
const hash = createHash("sha256").update(value).digest();
return round((hash[0] ?? 0) / 255);
}
function dot(left: readonly number[], right: readonly number[]): number {
return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
}
function softmax(scores: readonly number[]): number[] {
const max = Math.max(...scores);
const exp = scores.map((score) => Math.exp(score - max));
const sum = exp.reduce((total, value) => total + value, 0);
return exp.map((value) => value / Math.max(1e-9, sum));
}
function decisionKindFromDecision(decision: RoutingDecision): RoutingDecisionKind {
return decision.kind;
}
function decisionFromKind(kind: RoutingDecisionKind, input: RouterInput): RoutingDecision {
switch (kind) {
case "abstain":
return {
kind: "abstain",
reason:
input.classified.intent.kind === "off-topic-to-other"
? "off-topic-strong"
: "learned-router",
};
case "template-only":
if (input.classified.intent.kind === "acknowledgement") {
return { kind, templateKey: "acknowledgement" };
}
if (
input.derived.phase.kind === "completed" ||
input.request.trigger === "completed"
) {
return { kind, templateKey: "completed" };
}
return { kind, templateKey: "greeting" };
case "tier1-direct":
return { kind, skipPlanner: false };
case "tier0-only":
case "tier1-with-tools":
case "tier2-full":
return { kind };
}
}
export function extractRouterFeatures(input: RouterInput): RouterFeatureVector {
return {
classifyConfidence: clamp01(input.classified.confidence),
classifySource: input.classified.source === "llm" ? 1 : 0,
intentKind: input.classified.intent.kind,
messageLength: input.request.userMessage?.trim().length ?? 0,
phaseKind: input.derived.phase.kind,
responseModeTools: input.derived.responseMode === "llm-with-tools" ? 1 : 0,
sessionCompleted: input.session?.status === "completed" ? 1 : 0,
tier: input.derived.tier,
trigger: input.request.trigger,
};
}
export function encodeRouterFeatures(features: RouterFeatureVector): number[] {
return [
1,
features.classifyConfidence,
features.classifySource,
stableBucket(features.intentKind),
stableBucket(features.phaseKind),
stableBucket(features.trigger),
Math.min(500, features.messageLength) / 500,
features.tier / 2,
features.responseModeTools,
features.sessionCompleted,
];
}
export function trainLearnedRouterModel(
samples: RouterTrainingSample[],
options?: {
epochs?: number | undefined;
learningRate?: number | undefined;
l2?: number | undefined;
trainedAt?: string | undefined;
},
): LearnedRouterModel {
const weights = ROUTING_DECISION_KINDS.map(() =>
ROUTER_FEATURE_NAMES.map(() => 0),
);
const epochs = options?.epochs ?? 200;
const learningRate = options?.learningRate ?? 0.2;
const l2 = options?.l2 ?? 0.001;
for (let epoch = 0; epoch < epochs; epoch++) {
for (const sample of samples) {
const encoded = encodeRouterFeatures(sample.features);
const targetIndex = ROUTING_DECISION_KINDS.indexOf(sample.label);
if (targetIndex < 0) {
continue;
}
const probs = softmax(weights.map((row) => dot(row, encoded)));
for (let decisionIndex = 0; decisionIndex < weights.length; decisionIndex++) {
const row = weights[decisionIndex];
if (!row) {
continue;
}
const error = (decisionIndex === targetIndex ? 1 : 0) - (probs[decisionIndex] ?? 0);
for (let featureIndex = 0; featureIndex < row.length; featureIndex++) {
row[featureIndex] =
(row[featureIndex] ?? 0) +
learningRate *
(error * (encoded[featureIndex] ?? 0) -
l2 * (row[featureIndex] ?? 0));
}
}
}
}
return {
decisionKinds: [...ROUTING_DECISION_KINDS],
featureNames: [...ROUTER_FEATURE_NAMES],
trainedAt: options?.trainedAt ?? new Date().toISOString(),
version: 1,
weights: weights.map((row) => row.map(round)),
};
}
export function predictRoutingDecisionKind(
model: LearnedRouterModel,
features: RouterFeatureVector,
): RoutingDecisionKind {
const encoded = encodeRouterFeatures(features);
const scores = model.weights.map((row) => dot(row, encoded));
const bestIndex = scores.reduce(
(best, score, index) => (score > (scores[best] ?? Number.NEGATIVE_INFINITY) ? index : best),
0,
);
return model.decisionKinds[bestIndex] ?? "tier0-only";
}
export function decideLearnedRouting(
input: RouterInput,
model: LearnedRouterModel,
): RoutingDecision {
return decisionFromKind(
predictRoutingDecisionKind(model, extractRouterFeatures(input)),
input,
);
}
export function loadLearnedRouterModel(path: string | undefined): LearnedRouterModel | null {
if (!path?.trim()) {
return null;
}
const cached = modelCache.get(path);
if (cached !== undefined) {
return cached;
}
if (!existsSync(path)) {
modelCache.set(path, null);
return null;
}
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as LearnedRouterModel;
if (
parsed.version !== 1 ||
parsed.featureNames.length !== ROUTER_FEATURE_NAMES.length ||
parsed.decisionKinds.length !== ROUTING_DECISION_KINDS.length ||
parsed.weights.length !== ROUTING_DECISION_KINDS.length ||
parsed.weights.some((row) => row.length !== ROUTER_FEATURE_NAMES.length)
) {
modelCache.set(path, null);
return null;
}
modelCache.set(path, parsed);
return parsed;
} catch {
modelCache.set(path, null);
return null;
}
}
export function saveLearnedRouterModel(path: string, model: LearnedRouterModel): void {
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, `${JSON.stringify(model, null, 2)}\n`);
modelCache.set(path, model);
}
function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
if (
phase.kind !== "fresh" &&
phase.kind !== "nickname-pending" &&
phase.kind !== "photo-pending" &&
phase.kind !== "both-pending"
) {
return false;
}
return (
intent.kind === "nickname-question" ||
intent.kind === "photo-question" ||
intent.kind === "help-how" ||
intent.kind === "help-where"
);
}
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
const { classified, derived, request, session } = input;
const { confidence, intent } = classified;
const { phase } = derived;
if (intent.kind === "off-topic-to-other" && confidence > 0.9) {
return {
kind: "abstain",
reason: "off-topic-strong",
};
}
if (intent.kind === "greeting" && confidence > 0.95) {
return {
kind: "template-only",
templateKey: "greeting",
};
}
if (intent.kind === "acknowledgement" && confidence > 0.9) {
return {
kind: "template-only",
templateKey: "acknowledgement",
};
}
if (phase.kind === "completed" && intent.kind !== "status-question") {
return {
kind: "template-only",
templateKey: "completed",
};
}
if (request.trigger === "completed" || session?.status === "completed") {
return {
kind: "template-only",
templateKey: "completed",
};
}
if (
phase.kind === "nickname-rejected" ||
intent.kind === "discord-verify-question" ||
needsStepOneHelp(phase, intent)
) {
return {
kind: "tier1-with-tools",
};
}
if (phase.kind === "deadline-near") {
return {
kind: "tier1-direct",
skipPlanner: false,
};
}
return {
kind: "tier0-only",
};
}
export function decideHybridRouting(
input: RouterInput,
options?: {
learnedModel?: LearnedRouterModel | null | undefined;
},
): RoutingDecision {
if (!options?.learnedModel) {
return decidePreGenerationRouting(input);
}
const staticDecision = decidePreGenerationRouting(input);
if (staticDecision.kind === "abstain" || staticDecision.kind === "template-only") {
return staticDecision;
}
return decideLearnedRouting(input, options.learnedModel);
}
export function routerDecisionKind(decision: RoutingDecision): RoutingDecisionKind {
return decisionKindFromDecision(decision);
}
@@ -0,0 +1,55 @@
import { describe, expect, it } from "vitest";
import {
loadRuntimeFlagManifest,
parseRuntimeFlagManifest,
resolveRuntimeFlags,
type ResolvedRuntimeFlags,
} from "./runtime-flag-manifest.js";
const baseFlags: ResolvedRuntimeFlags = {
abstentionClarifyThreshold: 0.45,
abstentionEnabled: false,
abstentionThreshold: 0.35,
criticMaxIterations: 2,
programmaticToolCalling: false,
routerMode: "static",
toolLoopMaxHops: 3,
userMemoryEnabled: false,
};
describe("runtime flag manifests", () => {
it("loads built-in runtime variants", () => {
const manifest = loadRuntimeFlagManifest("abstention-on");
expect(manifest).toMatchObject({
variantId: "abstention-on",
flags: {
abstentionEnabled: true,
},
});
});
it("merges variant flags over base flags", () => {
const manifest = loadRuntimeFlagManifest("toolplan-on");
const resolved = resolveRuntimeFlags(baseFlags, manifest);
expect(resolved).toMatchObject({
abstentionEnabled: false,
programmaticToolCalling: true,
toolLoopMaxHops: 3,
});
});
it("rejects unknown flag keys", () => {
const manifest = parseRuntimeFlagManifest({
variantId: "bad",
description: "bad manifest",
flags: {
shellCommand: true,
},
});
expect(manifest).toBeNull();
});
});
@@ -0,0 +1,186 @@
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
export type RouterMode = "static" | "learned-v1" | "learned-v2";
export interface RuntimeFlagManifest {
description: string;
flags: Partial<RuntimeFlagSet>;
variantId: string;
}
export interface RuntimeFlagSet {
abstentionClarifyThreshold: number;
abstentionEnabled: boolean;
abstentionThreshold: number;
criticMaxIterations: number;
programmaticToolCalling: boolean;
routerMode: RouterMode;
toolLoopMaxHops: number;
userMemoryEnabled: boolean;
}
export type ResolvedRuntimeFlags = RuntimeFlagSet;
const runtimeManifestCache = new Map<string, RuntimeFlagManifest | null>();
const allowedFlagKeys = new Set<keyof RuntimeFlagSet>([
"abstentionClarifyThreshold",
"abstentionEnabled",
"abstentionThreshold",
"criticMaxIterations",
"programmaticToolCalling",
"routerMode",
"toolLoopMaxHops",
"userMemoryEnabled",
]);
const builtInManifests: Record<string, RuntimeFlagManifest> = {
"abstention-on": {
variantId: "abstention-on",
description: "Enable hybrid early abstention with default thresholds.",
flags: {
abstentionEnabled: true,
},
},
"toolplan-on": {
variantId: "toolplan-on",
description: "Enable planner-declared programmatic tool plans.",
flags: {
programmaticToolCalling: true,
toolLoopMaxHops: 3,
},
},
};
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function isRouterMode(value: unknown): value is RouterMode {
return (
value === "static" ||
value === "learned-v1" ||
value === "learned-v2"
);
}
function parseFlagValue(
key: keyof RuntimeFlagSet,
value: unknown,
): RuntimeFlagSet[keyof RuntimeFlagSet] | undefined {
switch (key) {
case "abstentionEnabled":
case "programmaticToolCalling":
case "userMemoryEnabled":
return typeof value === "boolean" ? value : undefined;
case "routerMode":
return isRouterMode(value) ? value : undefined;
case "abstentionClarifyThreshold":
case "abstentionThreshold":
return typeof value === "number" &&
Number.isFinite(value) &&
value >= 0 &&
value <= 1
? value
: undefined;
case "criticMaxIterations":
case "toolLoopMaxHops":
return typeof value === "number" && Number.isInteger(value) && value > 0
? value
: undefined;
}
}
export function parseRuntimeFlagManifest(
value: unknown,
expectedVariantId?: string,
): RuntimeFlagManifest | null {
const record = asRecord(value);
const variantId = record["variantId"];
const description = record["description"];
const rawFlags = asRecord(record["flags"]);
if (
typeof variantId !== "string" ||
!variantId.trim() ||
(expectedVariantId && variantId !== expectedVariantId) ||
typeof description !== "string" ||
!description.trim()
) {
return null;
}
const flags: Partial<RuntimeFlagSet> = {};
for (const [key, rawValue] of Object.entries(rawFlags)) {
if (!allowedFlagKeys.has(key as keyof RuntimeFlagSet)) {
return null;
}
const parsed = parseFlagValue(key as keyof RuntimeFlagSet, rawValue);
if (parsed === undefined) {
return null;
}
Object.assign(flags, { [key]: parsed });
}
return {
variantId,
description,
flags,
};
}
export function loadRuntimeFlagManifest(
variantId: string,
): RuntimeFlagManifest | null {
const normalized = variantId.trim();
if (!normalized || normalized === "control") {
return null;
}
const builtIn = builtInManifests[normalized];
if (builtIn) {
return builtIn;
}
if (runtimeManifestCache.has(normalized)) {
return runtimeManifestCache.get(normalized) ?? null;
}
const path = resolve(
process.cwd(),
"src",
"eval",
"proposals",
`${normalized}.runtime.json`,
);
if (!existsSync(path)) {
runtimeManifestCache.set(normalized, null);
return null;
}
try {
const parsed = parseRuntimeFlagManifest(
JSON.parse(readFileSync(path, "utf8")) as unknown,
normalized,
);
runtimeManifestCache.set(normalized, parsed);
return parsed;
} catch {
runtimeManifestCache.set(normalized, null);
return null;
}
}
export function resolveRuntimeFlags(
base: ResolvedRuntimeFlags,
variant?: RuntimeFlagManifest | null,
): ResolvedRuntimeFlags {
return {
...base,
...(variant?.flags ?? {}),
};
}
@@ -3,6 +3,8 @@ import { z } from "zod";
import { runWithTools } from "./tool-loop.js";
import type { Tool } from "@tools/tool";
import { ToolRegistry } from "@tools/registry";
import { createToolSearchTool } from "@tools/tool-search";
describe("runWithTools", () => {
it("executes a tool call and returns the model's final answer", async () => {
@@ -106,6 +108,81 @@ describe("runWithTools", () => {
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
});
it("expands searched tools for the next tool-call hop", async () => {
const toolOptionNames: string[][] = [];
const llmClientStub = {
chatCompletion: (
_messages: Array<Record<string, unknown>>,
options?: { tools?: Array<{ name: string }> },
) => {
toolOptionNames.push(options?.tools?.map((tool) => tool.name) ?? []);
if (toolOptionNames.length === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
},
});
}
if (toolOptionNames.length === 2) {
return Promise.resolve({
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "닉네임 형식을 다시 맞춰주세요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "닉네임 형식을 다시 맞춰주세요.",
},
});
},
};
const validateTool: Tool<{ nickname: string }, { valid: boolean }> = {
argsSchema: z.object({
nickname: z.string(),
}),
description: "닉네임 형식을 검사한다.",
execute: () => Promise.resolve({ valid: false }),
name: "validate-nickname",
resultSchema: z.object({
valid: z.boolean(),
}),
};
const registry = new ToolRegistry().register(validateTool);
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "닉네임 확인해줘" }],
toolContext: { toolRegistry: registry },
tools: [createToolSearchTool(registry)],
});
expect(result).toContain("닉네임");
expect(toolOptionNames[0]).toEqual(["search_tools"]);
expect(toolOptionNames[1]).toEqual(["search_tools", "validate-nickname"]);
});
it("preserves native tool call responses for follow-up turns", async () => {
const seenMessages: Array<Array<Record<string, unknown>>> = [];
@@ -216,4 +293,291 @@ describe("runWithTools", () => {
),
).toBe(true);
});
it("returns permanent tool failures to the model instead of throwing", async () => {
const seenMessages: Array<Array<Record<string, unknown>>> = [];
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
seenMessages.push(messages);
if (seenMessages.length === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
},
});
},
};
const tool: Tool<{ value: string }, { ok: boolean }> = {
name: "explode",
description: "항상 실패",
argsSchema: z.object({
value: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
}),
execute: () => Promise.reject(new Error("boom")),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toContain("기본 안내");
const toolMessage = seenMessages[1]?.find(
(message) => message.role === "tool",
);
expect(JSON.parse(String(toolMessage?.content))).toMatchObject({
errorKind: "permanent",
});
});
it("allows one self-correction hop when tool arguments are invalid", async () => {
let executeCount = 0;
const seenMessages: Array<Array<Record<string, unknown>>> = [];
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
seenMessages.push(messages);
if (seenMessages.length === 1) {
return Promise.resolve({
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
},
});
}
if (seenMessages.length === 2) {
return Promise.resolve({
content:
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "프리리님 기준으로 확인했어요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "프리리님 기준으로 확인했어요.",
},
});
},
};
const tool: Tool<{ name: string }, { ok: boolean }> = {
name: "needs-name",
description: "이름 필요",
argsSchema: z.object({
name: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
}),
execute: () => {
executeCount += 1;
return Promise.resolve({ ok: true });
},
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toContain("프리리님");
expect(executeCount).toBe(1);
expect(JSON.stringify(seenMessages[1])).toContain("schema");
});
it("retries transient tool errors once before continuing", async () => {
let executeCount = 0;
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
if (messages.some((message) => message.role === "tool")) {
return Promise.resolve({
content: "재시도 후 확인됐어요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "재시도 후 확인됐어요.",
},
});
}
return Promise.resolve({
content: '<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
},
});
},
};
const tool: Tool<{ value: string }, { ok: boolean }> = {
name: "flaky",
description: "일시 실패",
argsSchema: z.object({
value: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
}),
execute: () => {
executeCount += 1;
if (executeCount === 1) {
const error = new Error("timeout while calling dependency");
error.name = "AbortError";
return Promise.reject(error);
}
return Promise.resolve({ ok: true });
},
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toBe("재시도 후 확인됐어요.");
expect(executeCount).toBe(2);
});
it("grants one extra hop for precondition failures", async () => {
let executeCount = 0;
const llmClientStub = {
chatCompletion: () => {
if (executeCount === 0) {
return Promise.resolve({
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
},
});
}
if (executeCount === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content:
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "추가 확인까지 마쳤어요.",
endpoint: "mock://llm",
provider: "llama",
raw: { choices: [{ message: { content: "" } }] },
message: {
role: "assistant",
content: "추가 확인까지 마쳤어요.",
},
});
},
};
const tool: Tool<
{ value: string },
{ ok: boolean; errorKind?: string | undefined; message?: string | undefined }
> = {
name: "needs-session",
description: "세션 필요",
argsSchema: z.object({
value: z.string(),
}),
resultSchema: z.object({
ok: z.boolean(),
errorKind: z.string().optional(),
message: z.string().optional(),
}),
execute: () => {
executeCount += 1;
if (executeCount === 1) {
return Promise.resolve({
ok: false,
errorKind: "precondition-failed",
message: "세션이 없어요.",
});
}
return Promise.resolve({ ok: true });
},
};
const result = await runWithTools({
llmClient: llmClientStub as never,
maxHops: 1,
messages: [{ role: "user", content: "확인해줘" }],
toolContext: {},
tools: [tool],
});
expect(result).toContain("추가 확인");
expect(executeCount).toBe(2);
});
});
+256 -15
View File
@@ -3,7 +3,13 @@ import { getRequestLogger } from "@obs/context";
import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
import type { Tool, ToolContext } from "@tools/tool";
import type {
Tool,
ToolContext,
ToolErrorKind,
ToolExecutionFailure,
} from "@tools/tool";
import { getAgentToolLoopMaxHops, getAgentToolRetryMax } from "../../../config.js";
import { getActiveProcessTrace } from "./trace.js";
export interface ToolLoopOptions {
@@ -14,6 +20,7 @@ export interface ToolLoopOptions {
messages: ChatMessage[];
modelOverride?: string | undefined;
role?: ChatCompletionOptions["role"];
routingDecision?: ChatCompletionOptions["routingDecision"];
signal?: AbortSignal | undefined;
temperature?: number | undefined;
tier?: ChatCompletionOptions["tier"];
@@ -25,20 +32,25 @@ export async function runWithTools(
options: ToolLoopOptions,
): Promise<string | null> {
const logger = getRequestLogger();
const maxHops = options.maxHops ?? 2;
const maxHops = options.maxHops ?? getAgentToolLoopMaxHops();
const maxRetries = getAgentToolRetryMax();
const messages = options.messages.map((message) => ({ ...message }));
const activeTools = [...options.tools];
let allowNativeTools = true;
let preconditionBonusHops = 0;
let selfCorrectionHops = 0;
for (let hop = 0; hop <= maxHops; hop++) {
for (let hop = 0; hop <= maxHops + preconditionBonusHops; hop++) {
const response = await options.llmClient.chatCompletion(messages, {
deadlineMs: options.deadlineMs,
maxTokens: options.maxTokens,
modelOverride: options.modelOverride,
role: options.role,
routingDecision: options.routingDecision,
signal: options.signal,
temperature: options.temperature,
tier: options.tier,
tools: allowNativeTools ? options.tools : undefined,
tools: allowNativeTools ? activeTools : undefined,
});
if (!response) {
@@ -80,7 +92,7 @@ export async function runWithTools(
"[ToolLoop] Parsed tool calls",
);
if (hop === maxHops) {
if (hop === maxHops + preconditionBonusHops) {
messages.push(response.message);
messages.push({
role: "user",
@@ -94,9 +106,16 @@ export async function runWithTools(
messages.push(response.message);
for (const call of toolCalls) {
const tool = options.tools.find((candidate) => candidate.name === call.name);
const tool = activeTools.find((candidate) => candidate.name === call.name);
if (!tool) {
throw new Error(`Tool not registered for call: ${call.name}`);
const failure = makeToolFailure({
kind: "permanent",
message: `Tool not registered for call: ${call.name}`,
retryable: false,
toolName: call.name,
});
pushToolFailureMessage(messages, call, failure);
continue;
}
logger.debug(
@@ -110,30 +129,75 @@ export async function runWithTools(
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
let result: unknown;
let retried = false;
try {
const validated = tool.argsSchema.parse(call.arguments);
result = await withToolSpan(
tool.name,
result = await executeToolWithRetry(
() =>
withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
),
{
"tool.call.id": call.id,
"tool.name": tool.name,
maxRetries,
},
() => tool.execute(validated, options.toolContext),
);
).then((execution) => {
retried = execution.retried;
return execution.result;
});
const preconditionFailure = getPreconditionFailure(tool.name, result);
if (preconditionFailure) {
if (
preconditionFailure.kind === "precondition-failed" &&
preconditionBonusHops < 1
) {
preconditionBonusHops += 1;
}
traceStep?.finish({
args: call.arguments,
errorKind: preconditionFailure.kind,
name: tool.name,
ok: false,
retried,
resultSummary: preconditionFailure.message,
});
pushToolFailureMessage(messages, call, preconditionFailure, result);
continue;
}
traceStep?.finish({
args: call.arguments,
name: tool.name,
ok: true,
retried,
resultSummary: summarizeToolResult(result),
});
} catch (error) {
const failure = classifyToolError(tool.name, error);
traceStep?.finish({
args: call.arguments,
errorKind: failure.kind,
name: tool.name,
ok: false,
resultSummary: error instanceof Error ? error.message : String(error),
retried: failure.kind === "transient" && maxRetries > 0,
resultSummary: failure.message,
});
throw error;
pushToolFailureMessage(messages, call, failure);
if (failure.kind === "args-invalid" && selfCorrectionHops < 1) {
selfCorrectionHops += 1;
messages.push({
role: "user",
content:
`이전 ${tool.name} 도구 호출의 arguments가 schema와 맞지 않았어요. ` +
"도구 arguments를 고쳐서 한 번만 다시 호출하거나, 도구 없이 최종 답변하세요.",
});
break;
}
continue;
}
logger.debug(
@@ -145,6 +209,7 @@ export async function runWithTools(
"[ToolLoop] Tool call completed",
);
expandActiveToolsFromSearchResult(activeTools, options.toolContext, result);
messages.push({
role: "tool",
name: tool.name,
@@ -157,6 +222,182 @@ export async function runWithTools(
return null;
}
function expandActiveToolsFromSearchResult(
activeTools: Tool<unknown, unknown>[],
toolContext: ToolContext,
result: unknown,
): void {
const registry = toolContext.toolRegistry;
const tools = asRecord(result)["tools"];
if (!registry || !Array.isArray(tools)) {
return;
}
const existing = new Set(activeTools.map((tool) => tool.name));
for (const item of tools) {
const name = asRecord(item)["name"];
if (typeof name !== "string" || existing.has(name) || !registry.has(name)) {
continue;
}
const tool = registry.get(name);
activeTools.push(tool);
existing.add(tool.name);
}
}
function makeToolFailure(input: ToolExecutionFailure): ToolExecutionFailure {
return input;
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
function getErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function isTransientError(error: unknown): boolean {
const name = error instanceof Error ? error.name : "";
const message = getErrorMessage(error);
return (
name === "AbortError" ||
name === "FetchError" ||
/(?:timeout|timed out|ECONNRESET|ECONNREFUSED|EAI_AGAIN|ETIMEDOUT)/iu.test(
message,
)
);
}
async function executeToolWithRetry<R>(
execute: () => Promise<R>,
options: { maxRetries: number },
): Promise<{ result: R; retried: boolean }> {
let attempt = 0;
let retried = false;
while (true) {
try {
return {
result: await execute(),
retried,
};
} catch (error) {
if (!isTransientError(error) || attempt >= options.maxRetries) {
throw error;
}
attempt += 1;
retried = true;
await delay(200);
}
}
}
function classifyToolError(
toolName: string,
error: unknown,
): ToolExecutionFailure {
const message = getErrorMessage(error);
const isArgsInvalid =
error instanceof Error &&
(error.name === "ZodError" || error.constructor.name === "ZodError");
if (isArgsInvalid) {
return makeToolFailure({
kind: "args-invalid",
message,
retryable: true,
toolName,
details: error,
});
}
if (isTransientError(error)) {
return makeToolFailure({
kind: "transient",
message,
retryable: true,
toolName,
details: error,
});
}
return makeToolFailure({
kind: "permanent",
message,
retryable: false,
toolName,
details: error,
});
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function isToolErrorKind(value: unknown): value is ToolErrorKind {
return (
value === "args-invalid" ||
value === "transient" ||
value === "precondition-failed" ||
value === "permanent"
);
}
function getPreconditionFailure(
toolName: string,
result: unknown,
): ToolExecutionFailure | null {
const record = asRecord(result);
if (record["ok"] !== false) {
return null;
}
const kind = isToolErrorKind(record["errorKind"])
? record["errorKind"]
: isToolErrorKind(record["kind"])
? record["kind"]
: "precondition-failed";
const message =
typeof record["message"] === "string" && record["message"].trim()
? record["message"]
: "Tool precondition failed";
return makeToolFailure({
kind,
message,
retryable: kind === "transient" || kind === "args-invalid",
toolName,
details: result,
});
}
function pushToolFailureMessage(
messages: ChatMessage[],
call: { id?: string | undefined; name: string },
failure: ToolExecutionFailure,
result?: unknown,
): void {
messages.push({
role: "tool",
name: call.name,
tool_call_id: call.id,
content: JSON.stringify(
result ?? {
ok: false,
errorKind: failure.kind,
message: failure.message,
retryable: failure.retryable,
},
),
});
}
function summarizeToolResult(result: unknown): string {
try {
return JSON.stringify(result).slice(0, 200);
@@ -0,0 +1,145 @@
import { describe, expect, it } from "vitest";
import { z } from "zod";
import type { AgentRequest } from "@agent/types";
import type { Tool } from "@tools/tool";
import { runToolPlan } from "./tool-plan-runner.js";
import type { ToolPlan } from "./planner-schema.js";
function request(): AgentRequest {
return {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "확인해줘",
};
}
describe("runToolPlan", () => {
it("runs a small DAG and resolves arguments from request context", async () => {
const calls: string[] = [];
const validateTool: Tool<{ nickname: string }, { valid: boolean }> = {
name: "validate-nickname",
description: "닉네임 검사",
argsSchema: z.object({
nickname: z.string(),
}),
resultSchema: z.object({
valid: z.boolean(),
}),
execute: (args) => {
calls.push(`validate:${args.nickname}`);
return Promise.resolve({ valid: true });
},
};
const lookupTool: Tool<{ topic: string }, { text: string }> = {
name: "lookup-community-rule",
description: "규칙 조회",
argsSchema: z.object({
topic: z.string(),
}),
resultSchema: z.object({
text: z.string(),
}),
execute: (args) => {
calls.push(`lookup:${args.topic}`);
return Promise.resolve({ text: "하트 규칙" });
},
};
const plan: ToolPlan = {
composeWith: "all-then-llm",
steps: [
{
id: "validate",
tool: "validate-nickname",
argsFromContext: {
nickname: "$request.nickname",
},
},
{
id: "rule",
tool: "lookup-community-rule",
argsLiteral: {
topic: "heart",
},
dependsOn: ["validate"],
},
],
};
const result = await runToolPlan(
plan,
[validateTool, lookupTool],
{},
request(),
);
expect(result.failures).toEqual([]);
expect(result.results).toMatchObject({
validate: { valid: true },
rule: { text: "하트 규칙" },
});
expect(calls).toEqual(["validate:프리리/95/여/D", "lookup:heart"]);
});
it("reports missing dependencies without running tools", async () => {
const plan: ToolPlan = {
composeWith: "all-then-llm",
steps: [
{
id: "rule",
tool: "lookup-community-rule",
dependsOn: ["missing"],
},
],
};
const result = await runToolPlan(plan, [], {}, request());
expect(result.failures[0]).toMatchObject({
kind: "permanent",
toolName: "tool-plan",
});
expect(result.results).toEqual({});
});
it("marks fallbackToLlm when a failing step requests fallback", async () => {
const tool: Tool<{ topic: string }, { text: string }> = {
name: "lookup-community-rule",
description: "규칙 조회",
argsSchema: z.object({
topic: z.string(),
}),
resultSchema: z.object({
text: z.string(),
}),
execute: () => Promise.reject(new Error("rule service down")),
};
const plan: ToolPlan = {
composeWith: "all-then-llm",
steps: [
{
id: "rule",
tool: "lookup-community-rule",
argsLiteral: {
topic: "heart",
},
on: {
error: "fallback-llm",
},
},
],
};
const result = await runToolPlan(plan, [tool], {}, request());
expect(result.fallbackToLlm).toBe(true);
expect(result.failures[0]).toMatchObject({
kind: "permanent",
toolName: "lookup-community-rule",
});
});
});
@@ -0,0 +1,417 @@
import type { AgentRequest } from "@agent/types";
import { withToolSpan } from "@obs/spans";
import type {
Tool,
ToolContext,
ToolErrorKind,
ToolExecutionFailure,
} from "@tools/tool";
import type { ToolPlan, ToolPlanStep } from "./planner-schema.js";
import { getActiveProcessTrace } from "./trace.js";
export interface ToolPlanResult {
durationMs: number;
failures: ToolExecutionFailure[];
fallbackToLlm: boolean;
results: Record<string, unknown>;
}
interface OrderedPlan {
failures: ToolExecutionFailure[];
levels: ToolPlanStep[][];
}
function getErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function isTransientError(error: unknown): boolean {
const name = error instanceof Error ? error.name : "";
const message = getErrorMessage(error);
return (
name === "AbortError" ||
name === "FetchError" ||
/(?:timeout|timed out|ECONNRESET|ECONNREFUSED|EAI_AGAIN|ETIMEDOUT)/iu.test(
message,
)
);
}
function classifyError(toolName: string, error: unknown): ToolExecutionFailure {
const message = getErrorMessage(error);
const isArgsInvalid =
error instanceof Error &&
(error.name === "ZodError" || error.constructor.name === "ZodError");
if (isArgsInvalid) {
return {
kind: "args-invalid",
message,
retryable: true,
toolName,
details: error,
};
}
if (isTransientError(error)) {
return {
kind: "transient",
message,
retryable: true,
toolName,
details: error,
};
}
return {
kind: "permanent",
message,
retryable: false,
toolName,
details: error,
};
}
function makeFailure(input: {
details?: unknown;
kind: ToolErrorKind;
message: string;
retryable?: boolean;
toolName: string;
}): ToolExecutionFailure {
return {
kind: input.kind,
message: input.message,
retryable: input.retryable ?? false,
toolName: input.toolName,
...(input.details === undefined ? {} : { details: input.details }),
};
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function isToolErrorKind(value: unknown): value is ToolErrorKind {
return (
value === "args-invalid" ||
value === "transient" ||
value === "precondition-failed" ||
value === "permanent"
);
}
function getStructuredFailure(
step: ToolPlanStep,
result: unknown,
): ToolExecutionFailure | null {
const record = asRecord(result);
if (record["ok"] !== false) {
return null;
}
const kind = isToolErrorKind(record["errorKind"])
? record["errorKind"]
: isToolErrorKind(record["kind"])
? record["kind"]
: "precondition-failed";
const message =
typeof record["message"] === "string" && record["message"].trim()
? record["message"]
: "Tool precondition failed";
return makeFailure({
details: {
result,
stepId: step.id,
},
kind,
message,
retryable: kind === "args-invalid" || kind === "transient",
toolName: step.tool,
});
}
function parsePath(path: string): string[] {
const normalized = path.trim().replace(/^\$/u, "");
return normalized
.split(".")
.flatMap((segment) => {
const bracketIndex = segment.indexOf("[");
if (bracketIndex === -1 || !segment.endsWith("]")) {
return [segment];
}
const key = segment.slice(0, bracketIndex);
const index = segment.slice(bracketIndex + 1, -1);
return key && index ? [key, index] : [segment];
})
.filter(Boolean);
}
function readPath(root: unknown, path: string): unknown {
let current = root;
for (const segment of parsePath(path)) {
if (Array.isArray(current)) {
const index = Number(segment);
if (!Number.isInteger(index)) {
throw new Error(`Invalid array index in path: ${path}`);
}
current = current[index];
continue;
}
const record = asRecord(current);
if (!(segment in record)) {
throw new Error(`Missing context path: ${path}`);
}
current = record[segment];
}
return current;
}
function buildArgs(input: {
ctx: ToolContext;
request: AgentRequest;
results: Record<string, unknown>;
step: ToolPlanStep;
}): Record<string, unknown> {
const args: Record<string, unknown> = {
...(input.step.argsLiteral ?? {}),
};
const roots = {
context: input.ctx,
request: input.request,
results: input.results,
session: input.ctx.session ?? null,
tool: input.results,
};
for (const [targetKey, sourcePath] of Object.entries(
input.step.argsFromContext ?? {},
)) {
args[targetKey] = readPath(roots, sourcePath);
}
return args;
}
function makePlanFailure(message: string, details?: unknown): ToolExecutionFailure {
return makeFailure({
details,
kind: "permanent",
message,
toolName: "tool-plan",
});
}
function orderPlan(plan: ToolPlan): OrderedPlan {
const failures: ToolExecutionFailure[] = [];
const byId = new Map<string, ToolPlanStep>();
for (const step of plan.steps) {
if (byId.has(step.id)) {
failures.push(makePlanFailure(`Duplicate toolPlan step id: ${step.id}`));
continue;
}
byId.set(step.id, step);
}
for (const step of plan.steps) {
for (const dependency of step.dependsOn ?? []) {
if (!byId.has(dependency)) {
failures.push(
makePlanFailure(`Missing dependency ${dependency} for step ${step.id}`),
);
}
}
}
if (failures.length > 0) {
return { failures, levels: [] };
}
const remaining = new Map(byId);
const completed = new Set<string>();
const levels: ToolPlanStep[][] = [];
while (remaining.size > 0) {
const ready = [...remaining.values()].filter((step) =>
(step.dependsOn ?? []).every((dependency) => completed.has(dependency)),
);
if (ready.length === 0) {
return {
failures: [makePlanFailure("Cyclic toolPlan dependencies detected")],
levels,
};
}
levels.push(ready);
for (const step of ready) {
remaining.delete(step.id);
completed.add(step.id);
}
}
return { failures, levels };
}
async function runStep(input: {
ctx: ToolContext;
request: AgentRequest;
results: Record<string, unknown>;
step: ToolPlanStep;
tools: readonly Tool<unknown, unknown>[];
}): Promise<ToolExecutionFailure | null> {
const tool = input.tools.find((candidate) => candidate.name === input.step.tool);
if (!tool) {
return makeFailure({
details: { stepId: input.step.id },
kind: "permanent",
message: `Tool not registered for step: ${input.step.tool}`,
toolName: input.step.tool,
});
}
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
let args: Record<string, unknown> = {};
try {
args = buildArgs(input);
const validated = tool.argsSchema.parse(args);
const result = await withToolSpan(
tool.name,
{
"tool.name": tool.name,
"tool.plan.step_id": input.step.id,
},
() => tool.execute(validated, input.ctx),
);
const structuredFailure = getStructuredFailure(input.step, result);
if (structuredFailure) {
traceStep?.finish({
args,
errorKind: structuredFailure.kind,
name: tool.name,
ok: false,
resultSummary: structuredFailure.message,
});
return structuredFailure;
}
input.results[input.step.id] = result;
traceStep?.finish({
args,
name: tool.name,
ok: true,
resultSummary: summarizeToolResult(result),
});
return null;
} catch (error) {
const failure = classifyError(tool.name, error);
traceStep?.finish({
args,
errorKind: failure.kind,
name: tool.name,
ok: false,
resultSummary: failure.message,
});
return failure;
}
}
function shouldFallback(step: ToolPlanStep): boolean {
return step.on?.error === "fallback-llm";
}
async function runLevel(input: {
ctx: ToolContext;
failures: ToolExecutionFailure[];
request: AgentRequest;
results: Record<string, unknown>;
steps: ToolPlanStep[];
tools: readonly Tool<unknown, unknown>[];
}): Promise<{ abort: boolean; fallbackToLlm: boolean }> {
let abort = false;
let fallbackToLlm = false;
for (let index = 0; index < input.steps.length; index += 2) {
const chunk = input.steps.slice(index, index + 2);
const failures = await Promise.all(
chunk.map((step) =>
runStep({
ctx: input.ctx,
request: input.request,
results: input.results,
step,
tools: input.tools,
}).then((failure) => ({ failure, step })),
),
);
for (const { failure, step } of failures) {
if (!failure) {
continue;
}
input.failures.push(failure);
if (shouldFallback(step)) {
fallbackToLlm = true;
}
if (step.on?.error === "abort" || !step.on?.error) {
abort = true;
}
}
if (abort || fallbackToLlm) {
return { abort, fallbackToLlm };
}
}
return { abort, fallbackToLlm };
}
export async function runToolPlan(
plan: ToolPlan,
tools: readonly Tool<unknown, unknown>[],
ctx: ToolContext,
request: AgentRequest,
): Promise<ToolPlanResult> {
const startedAt = Date.now();
const results: Record<string, unknown> = {};
const ordered = orderPlan(plan);
const failures = [...ordered.failures];
let fallbackToLlm = false;
for (const level of ordered.levels) {
const levelResult = await runLevel({
ctx,
failures,
request,
results,
steps: level,
tools,
});
fallbackToLlm = fallbackToLlm || levelResult.fallbackToLlm;
if (levelResult.abort || levelResult.fallbackToLlm) {
break;
}
}
return {
durationMs: Math.max(0, Date.now() - startedAt),
failures,
fallbackToLlm,
results,
};
}
function summarizeToolResult(result: unknown): string {
try {
return JSON.stringify(result).slice(0, 200);
} catch {
return String(result).slice(0, 200);
}
}
+17
View File
@@ -1,5 +1,9 @@
import { AsyncLocalStorage } from "node:async_hooks";
import type { AbstentionDecision } from "./abstention.js";
import type { RoutingDecision } from "./router.js";
import type { ToolErrorKind } from "@tools/tool";
export type ProcessTraceStepName =
| "classify"
| "critic"
@@ -19,6 +23,7 @@ export interface ProcessTraceStep<T> {
export interface ProcessTraceSnapshot {
steps: {
classify?: ProcessTraceStep<{
abstention?: AbstentionDecision | undefined;
confidence: number;
intent: string;
source: "regex" | "llm";
@@ -26,6 +31,7 @@ export interface ProcessTraceSnapshot {
derive?: ProcessTraceStep<{
phase: string;
responseMode: string;
routingDecision?: RoutingDecision | undefined;
shouldRespond: boolean;
tier: 0 | 1 | 2;
}> | undefined;
@@ -33,14 +39,18 @@ export interface ProcessTraceSnapshot {
approach: string;
mustAvoid: string[];
mustInclude: string[];
requestResources?: string[] | undefined;
suggestedTools: string[];
targetLengthChars: number;
toolPlanStepCount?: number | undefined;
}> | undefined;
toolCalls?: Array<
ProcessTraceStep<{
args: unknown;
errorKind?: ToolErrorKind | undefined;
name: string;
ok: boolean;
retried?: boolean | undefined;
resultSummary: string;
}>
> | undefined;
@@ -50,8 +60,15 @@ export interface ProcessTraceSnapshot {
tokensOut: number;
}> | undefined;
critic?: ProcessTraceStep<{
abstained?: boolean | undefined;
decision: "keep" | "rewrite";
iterations?: number | undefined;
rationale: string;
rubricScores?: {
accuracy: number;
scope: number;
tone: number;
} | undefined;
}> | undefined;
validate?: ProcessTraceStep<{
reasons: string[];