Add skill governance and runtime routing
This commit is contained in:
1 parent
a900016be4
commit
221e9391e0
71 files changed
+7057
-102
No files matched your search
@@ -0,0 +1,137 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import { decideEarlyAbstention } from "./abstention.js";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
Intent,
|
||||
} from "../state/onboarding-state.js";
|
||||
import type { Blackboard } from "../types.js";
|
||||
|
||||
function request(overrides?: Partial<AgentRequest>): AgentRequest {
|
||||
return {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "테스터",
|
||||
trigger: "user-message",
|
||||
userMessage: "뭐죠",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function classified(
|
||||
intent: Intent["kind"],
|
||||
confidence: number,
|
||||
): ClassifiedIntent {
|
||||
return {
|
||||
intent: { kind: intent } as Intent,
|
||||
confidence,
|
||||
source: "llm",
|
||||
};
|
||||
}
|
||||
|
||||
function derived(intent: Intent["kind"]): DerivedState {
|
||||
return {
|
||||
phase: { kind: "fresh" },
|
||||
intent: { kind: intent } as Intent,
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function blackboard(overrides?: Partial<Blackboard>): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const thresholds = {
|
||||
abstain: 0.35,
|
||||
clarify: 0.45,
|
||||
};
|
||||
|
||||
describe("decideEarlyAbstention", () => {
|
||||
it("returns no-response for high-confidence off-topic intent", () => {
|
||||
const decision = decideEarlyAbstention({
|
||||
blackboard: null,
|
||||
classified: classified("off-topic-to-other", 0.95),
|
||||
derived: derived("off-topic-to-other"),
|
||||
request: request(),
|
||||
thresholds,
|
||||
});
|
||||
|
||||
expect(decision.action.kind).toBe("no-response");
|
||||
expect(decision.reason).toBe("off-topic-strong");
|
||||
});
|
||||
|
||||
it("asks for clarification on low-confidence early unknown turns", () => {
|
||||
const decision = decideEarlyAbstention({
|
||||
blackboard: blackboard(),
|
||||
classified: classified("unknown", 0.4),
|
||||
derived: derived("unknown"),
|
||||
request: request(),
|
||||
thresholds,
|
||||
});
|
||||
|
||||
expect(decision.action).toEqual({
|
||||
kind: "clarify",
|
||||
templateKey: "abstain-clarify-default",
|
||||
});
|
||||
expect(decision.reason).toBe("low-confidence-early-turn");
|
||||
});
|
||||
|
||||
it("returns no-response for repeated low-confidence intent loops", () => {
|
||||
const decision = decideEarlyAbstention({
|
||||
blackboard: blackboard({
|
||||
lastIntent: "unknown",
|
||||
sameIntentStreak: 2,
|
||||
}),
|
||||
classified: classified("unknown", 0.2),
|
||||
derived: derived("unknown"),
|
||||
request: request({ context: { turnIndex: 3 } }),
|
||||
thresholds,
|
||||
});
|
||||
|
||||
expect(decision.action.kind).toBe("no-response");
|
||||
expect(decision.reason).toBe("low-confidence-repeated");
|
||||
expect(decision.features.sameIntentStreak).toBe(3);
|
||||
});
|
||||
|
||||
it("prioritizes flagged confusion when low confidence and issues are present", () => {
|
||||
const decision = decideEarlyAbstention({
|
||||
blackboard: blackboard({
|
||||
flaggedIssues: ["닉네임 형식 반복 오류"],
|
||||
}),
|
||||
classified: classified("unknown", 0.3),
|
||||
derived: derived("unknown"),
|
||||
request: request({ context: { turnIndex: 2 } }),
|
||||
thresholds,
|
||||
});
|
||||
|
||||
expect(decision.action.kind).toBe("no-response");
|
||||
expect(decision.reason).toBe("flagged-confused");
|
||||
});
|
||||
|
||||
it("proceeds on clarify threshold boundary", () => {
|
||||
const decision = decideEarlyAbstention({
|
||||
blackboard: blackboard(),
|
||||
classified: classified("unknown", thresholds.clarify),
|
||||
derived: derived("unknown"),
|
||||
request: request(),
|
||||
thresholds,
|
||||
});
|
||||
|
||||
expect(decision.action.kind).toBe("proceed");
|
||||
expect(decision.reason).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
Intent,
|
||||
} from "../state/onboarding-state.js";
|
||||
import type { Blackboard } from "../types.js";
|
||||
|
||||
export type AbstentionAction =
|
||||
| { kind: "proceed" }
|
||||
| { kind: "clarify"; templateKey: string }
|
||||
| { kind: "no-response" };
|
||||
|
||||
export interface AbstentionDecision {
|
||||
action: AbstentionAction;
|
||||
reason:
|
||||
| "low-confidence-early-turn"
|
||||
| "low-confidence-repeated"
|
||||
| "off-topic-strong"
|
||||
| "flagged-confused"
|
||||
| null;
|
||||
score: number;
|
||||
features: {
|
||||
classifyConfidence: number;
|
||||
intentKind: Intent["kind"];
|
||||
turnIndex: number;
|
||||
sameIntentStreak: number;
|
||||
flaggedIssues: number;
|
||||
};
|
||||
}
|
||||
|
||||
function clampRatio(value: number): number {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function roundRatio(value: number): number {
|
||||
return Number(clampRatio(value).toFixed(3));
|
||||
}
|
||||
|
||||
function getExplicitTurnIndex(request: AgentRequest): number | null {
|
||||
const value = request.context?.["turnIndex"];
|
||||
return typeof value === "number" && Number.isFinite(value) && value >= 0
|
||||
? Math.floor(value)
|
||||
: null;
|
||||
}
|
||||
|
||||
function inferTurnIndex(request: AgentRequest, blackboard: Blackboard | null): number {
|
||||
const explicit = getExplicitTurnIndex(request);
|
||||
if (explicit !== null) {
|
||||
return explicit;
|
||||
}
|
||||
|
||||
if (!blackboard?.lastIntent) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return blackboard.toneMemory.greetedAt ? 2 : 1;
|
||||
}
|
||||
|
||||
function getSameIntentStreak(
|
||||
intentKind: Intent["kind"],
|
||||
blackboard: Blackboard | null,
|
||||
): number {
|
||||
if (!blackboard || blackboard.lastIntent !== intentKind) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return Math.max(2, (blackboard.sameIntentStreak ?? 1) + 1);
|
||||
}
|
||||
|
||||
function computeScore(input: {
|
||||
confidence: number;
|
||||
flaggedIssues: number;
|
||||
intentKind: Intent["kind"];
|
||||
sameIntentStreak: number;
|
||||
turnIndex: number;
|
||||
}): number {
|
||||
const lowConfidence = 1 - input.confidence;
|
||||
const offTopicBoost = input.intentKind === "off-topic-to-other" ? 0.25 : 0;
|
||||
const flaggedBoost = input.flaggedIssues > 0 ? 0.2 : 0;
|
||||
const repeatedBoost = input.sameIntentStreak >= 3 ? 0.15 : 0;
|
||||
const earlyBoost = input.turnIndex <= 1 ? 0.05 : 0;
|
||||
|
||||
return roundRatio(
|
||||
lowConfidence + offTopicBoost + flaggedBoost + repeatedBoost + earlyBoost,
|
||||
);
|
||||
}
|
||||
|
||||
function makeDecision(
|
||||
features: AbstentionDecision["features"],
|
||||
action: AbstentionAction,
|
||||
reason: AbstentionDecision["reason"],
|
||||
): AbstentionDecision {
|
||||
return {
|
||||
action,
|
||||
reason,
|
||||
score: computeScore({
|
||||
confidence: features.classifyConfidence,
|
||||
flaggedIssues: features.flaggedIssues,
|
||||
intentKind: features.intentKind,
|
||||
sameIntentStreak: features.sameIntentStreak,
|
||||
turnIndex: features.turnIndex,
|
||||
}),
|
||||
features,
|
||||
};
|
||||
}
|
||||
|
||||
export function decideEarlyAbstention(input: {
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState;
|
||||
blackboard: Blackboard | null;
|
||||
request: AgentRequest;
|
||||
thresholds: {
|
||||
abstain: number;
|
||||
clarify: number;
|
||||
};
|
||||
}): AbstentionDecision {
|
||||
const confidence = clampRatio(input.classified.confidence);
|
||||
const intentKind = input.classified.intent.kind;
|
||||
const features = {
|
||||
classifyConfidence: confidence,
|
||||
intentKind,
|
||||
turnIndex: inferTurnIndex(input.request, input.blackboard),
|
||||
sameIntentStreak: getSameIntentStreak(intentKind, input.blackboard),
|
||||
flaggedIssues: input.blackboard?.flaggedIssues.length ?? 0,
|
||||
};
|
||||
|
||||
if (intentKind === "off-topic-to-other" && confidence > 0.9) {
|
||||
return makeDecision(features, { kind: "no-response" }, "off-topic-strong");
|
||||
}
|
||||
|
||||
if (
|
||||
(features.flaggedIssues > 0 || features.sameIntentStreak >= 3) &&
|
||||
confidence < input.thresholds.abstain
|
||||
) {
|
||||
return makeDecision(
|
||||
features,
|
||||
{ kind: "no-response" },
|
||||
features.flaggedIssues > 0
|
||||
? "flagged-confused"
|
||||
: "low-confidence-repeated",
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
features.turnIndex <= 1 &&
|
||||
confidence < input.thresholds.clarify &&
|
||||
(intentKind === "unknown" || intentKind === "chitchat")
|
||||
) {
|
||||
return makeDecision(
|
||||
features,
|
||||
{ kind: "clarify", templateKey: "abstain-clarify-default" },
|
||||
"low-confidence-early-turn",
|
||||
);
|
||||
}
|
||||
|
||||
return makeDecision(features, { kind: "proceed" }, null);
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
|
||||
function derived(overrides?: Partial<DerivedState>): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: { kind: "help-how" },
|
||||
phase: { kind: "photo-pending" },
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 256,
|
||||
tier: 2,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const messages: ChatMessage[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: "하트 인증은 어떻게 해요?",
|
||||
},
|
||||
];
|
||||
|
||||
describe("OnboardingCritic", () => {
|
||||
afterEach(() => {
|
||||
delete process.env["AGENT_CRITIC_MAX_ITERATIONS"];
|
||||
});
|
||||
|
||||
it("iteratively rewrites and then keeps the improved draft", async () => {
|
||||
const calls: Array<{ content: string; options: Record<string, unknown> }> = [];
|
||||
const llmClient = {
|
||||
chatCompletion: (
|
||||
nextMessages: ChatMessage[],
|
||||
options: Record<string, unknown>,
|
||||
) => {
|
||||
calls.push({
|
||||
content: nextMessages.at(-1)?.content ?? "",
|
||||
options,
|
||||
});
|
||||
return Promise.resolve({
|
||||
content:
|
||||
calls.length === 1
|
||||
? JSON.stringify({
|
||||
decision: "rewrite",
|
||||
rationale: "too broad",
|
||||
rubricScores: { accuracy: 0.6, scope: 0.4, tone: 0.8 },
|
||||
text: "하트 인증 화면 원본 스크린샷만 올려주시면 돼요.",
|
||||
})
|
||||
: JSON.stringify({
|
||||
decision: "keep",
|
||||
rationale: "good",
|
||||
rubricScores: { accuracy: 0.9, scope: 0.9, tone: 0.8 },
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
message: { role: "assistant", content: "{}" },
|
||||
provider: "llama",
|
||||
raw: {},
|
||||
});
|
||||
},
|
||||
};
|
||||
const critic = new OnboardingCritic(llmClient as never);
|
||||
|
||||
const result = await critic.critiqueIterative(
|
||||
"여러 절차를 모두 진행하세요.",
|
||||
derived(),
|
||||
messages,
|
||||
{ maxIterations: 2 },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
abstained: false,
|
||||
changed: true,
|
||||
iterations: 2,
|
||||
text: "하트 인증 화면 원본 스크린샷만 올려주시면 돼요.",
|
||||
});
|
||||
expect(result.rubricScores.accuracy).toBe(0.9);
|
||||
expect(calls[0]?.options).toMatchObject({ jsonMode: true, role: "critic" });
|
||||
});
|
||||
|
||||
it("can abstain from unsafe drafts", async () => {
|
||||
const llmClient = {
|
||||
chatCompletion: () =>
|
||||
Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
decision: "abstain",
|
||||
rationale: "off topic",
|
||||
rubricScores: { accuracy: 0.1, scope: 0.1, tone: 0.5 },
|
||||
text: "[NO_RESPONSE]",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
message: { role: "assistant", content: "{}" },
|
||||
provider: "llama",
|
||||
raw: {},
|
||||
}),
|
||||
};
|
||||
const critic = new OnboardingCritic(llmClient as never);
|
||||
|
||||
const result = await critic.critiqueIterative(
|
||||
"상관없는 답변",
|
||||
derived(),
|
||||
messages,
|
||||
{ maxIterations: 2 },
|
||||
);
|
||||
|
||||
expect(result.abstained).toBe(true);
|
||||
expect(result.text).toBe("[NO_RESPONSE]");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
getCriticMaxTokens,
|
||||
getLlmTemperature,
|
||||
@@ -13,6 +14,29 @@ export interface CritiqueResult {
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface CritiqueResultV2 extends CritiqueResult {
|
||||
abstained: boolean;
|
||||
iterations: number;
|
||||
prmScoreAfter?: number | undefined;
|
||||
prmScoreBefore?: number | undefined;
|
||||
rubricScores: {
|
||||
accuracy: number;
|
||||
scope: number;
|
||||
tone: number;
|
||||
};
|
||||
}
|
||||
|
||||
const iterativeCriticSchema = z.object({
|
||||
decision: z.enum(["keep", "rewrite", "abstain"]),
|
||||
rationale: z.string().trim().max(300).optional(),
|
||||
rubricScores: z.object({
|
||||
accuracy: z.number().min(0).max(1),
|
||||
scope: z.number().min(0).max(1),
|
||||
tone: z.number().min(0).max(1),
|
||||
}),
|
||||
text: z.string().trim().max(1_000).optional(),
|
||||
});
|
||||
|
||||
function serializeMessages(messages: ChatMessage[]): string {
|
||||
return messages
|
||||
.map((message) => `[${message.role}] ${message.content}`)
|
||||
@@ -47,6 +71,67 @@ function buildCriticMessages(
|
||||
];
|
||||
}
|
||||
|
||||
function buildIterativeCriticMessages(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
iteration: number,
|
||||
): ChatMessage[] {
|
||||
return [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"당신은 온보딩 응답을 반복 검토하는 critic입니다.",
|
||||
"반드시 JSON object만 출력하세요.",
|
||||
"decision은 keep, rewrite, abstain 중 하나입니다.",
|
||||
"rewrite이면 text에 유저에게 바로 보낼 최종 한국어 답변만 넣으세요.",
|
||||
"abstain이면 text는 [NO_RESPONSE]로 두세요.",
|
||||
"rubricScores는 accuracy, scope, tone을 0..1 사이 숫자로 채우세요.",
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}`,
|
||||
`[ITERATION] ${iteration}`,
|
||||
"[CONVERSATION]",
|
||||
serializeMessages(messages),
|
||||
"[DRAFT]",
|
||||
draft,
|
||||
"",
|
||||
'JSON schema: {"decision":"keep|rewrite|abstain","text":"...","rubricScores":{"accuracy":0.8,"scope":0.8,"tone":0.8},"rationale":"..."}',
|
||||
].join("\n\n"),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function scoreRubric(scores: CritiqueResultV2["rubricScores"]): number {
|
||||
return scores.accuracy + scores.scope + scores.tone;
|
||||
}
|
||||
|
||||
function emptyRubric(): CritiqueResultV2["rubricScores"] {
|
||||
return {
|
||||
accuracy: 0,
|
||||
scope: 0,
|
||||
tone: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function parseIterativeCriticResponse(
|
||||
content: string | undefined,
|
||||
): z.infer<typeof iterativeCriticSchema> | null {
|
||||
if (!content?.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = iterativeCriticSchema.safeParse(JSON.parse(content));
|
||||
return parsed.success ? parsed.data : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class OnboardingCritic {
|
||||
constructor(private readonly llmClient: LlmClient) {}
|
||||
|
||||
@@ -90,4 +175,111 @@ export class OnboardingCritic {
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
async critiqueIterative(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: {
|
||||
deadlineMs?: number | undefined;
|
||||
maxIterations: number;
|
||||
},
|
||||
): Promise<CritiqueResultV2> {
|
||||
const traceStep = getActiveProcessTrace()?.startStep("critic");
|
||||
const maxIterations = Math.max(1, options.maxIterations);
|
||||
let current = draft;
|
||||
let changed = false;
|
||||
let bestRubric = emptyRubric();
|
||||
let bestScore = -1;
|
||||
let rationale = "max iterations reached";
|
||||
|
||||
for (let iteration = 1; iteration <= maxIterations; iteration++) {
|
||||
if (options.deadlineMs !== undefined && options.deadlineMs <= 0) {
|
||||
rationale = "deadline exceeded";
|
||||
break;
|
||||
}
|
||||
|
||||
const response = await this.llmClient.chatCompletion(
|
||||
buildIterativeCriticMessages(current, derived, messages, iteration),
|
||||
{
|
||||
deadlineMs: options.deadlineMs,
|
||||
jsonMode: true,
|
||||
maxTokens: getCriticMaxTokens(),
|
||||
modelOverride: getTier2CriticModel(),
|
||||
role: "critic",
|
||||
temperature: Math.max(0, getLlmTemperature() - 0.15),
|
||||
tier: 2,
|
||||
},
|
||||
);
|
||||
const parsed = parseIterativeCriticResponse(response?.content);
|
||||
if (!parsed) {
|
||||
rationale = "invalid critic response";
|
||||
break;
|
||||
}
|
||||
|
||||
const nextScore = scoreRubric(parsed.rubricScores);
|
||||
if (nextScore >= bestScore) {
|
||||
bestRubric = parsed.rubricScores;
|
||||
bestScore = nextScore;
|
||||
}
|
||||
rationale = parsed.rationale ?? parsed.decision;
|
||||
|
||||
if (parsed.decision === "abstain") {
|
||||
traceStep?.finish({
|
||||
abstained: true,
|
||||
decision: "rewrite",
|
||||
iterations: iteration,
|
||||
rationale,
|
||||
rubricScores: bestRubric,
|
||||
});
|
||||
return {
|
||||
abstained: true,
|
||||
changed: true,
|
||||
iterations: iteration,
|
||||
rubricScores: bestRubric,
|
||||
text: "[NO_RESPONSE]",
|
||||
};
|
||||
}
|
||||
|
||||
if (parsed.decision === "keep") {
|
||||
traceStep?.finish({
|
||||
abstained: false,
|
||||
decision: changed ? "rewrite" : "keep",
|
||||
iterations: iteration,
|
||||
rationale,
|
||||
rubricScores: bestRubric,
|
||||
});
|
||||
return {
|
||||
abstained: false,
|
||||
changed,
|
||||
iterations: iteration,
|
||||
rubricScores: bestRubric,
|
||||
text: current,
|
||||
};
|
||||
}
|
||||
|
||||
const nextText = parsed.text?.trim();
|
||||
if (!nextText || nextText === current || nextScore < bestScore) {
|
||||
break;
|
||||
}
|
||||
|
||||
current = nextText;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
traceStep?.finish({
|
||||
abstained: false,
|
||||
decision: changed ? "rewrite" : "keep",
|
||||
iterations: maxIterations,
|
||||
rationale,
|
||||
rubricScores: bestRubric,
|
||||
});
|
||||
return {
|
||||
abstained: false,
|
||||
changed,
|
||||
iterations: maxIterations,
|
||||
rubricScores: bestRubric,
|
||||
text: current,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { OnboardingOrchestrator } from "./orchestrator.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import type { Tool } from "@tools/tool";
|
||||
|
||||
function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
|
||||
return {
|
||||
@@ -17,6 +19,11 @@ function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
|
||||
}
|
||||
|
||||
describe("OnboardingOrchestrator", () => {
|
||||
afterEach(() => {
|
||||
delete process.env["AGENT_PROGRAMMATIC_TOOL_CALLING"];
|
||||
delete process.env["AGENT_SKILL_MD_MODE"];
|
||||
});
|
||||
|
||||
it("uses direct response path for tier 0", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
@@ -107,6 +114,165 @@ describe("OnboardingOrchestrator", () => {
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[PLAN]");
|
||||
});
|
||||
|
||||
it("runs a planner toolPlan before the responder when enabled", async () => {
|
||||
process.env["AGENT_PROGRAMMATIC_TOOL_CALLING"] = "true";
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "validate-then-explain",
|
||||
targetPhase: "nickname-rejected",
|
||||
mustInclude: ["검사 결과"],
|
||||
mustAvoid: [],
|
||||
suggestedTools: ["validate-nickname"],
|
||||
toolPlan: {
|
||||
composeWith: "all-then-llm",
|
||||
steps: [
|
||||
{
|
||||
id: "validate",
|
||||
tool: "validate-nickname",
|
||||
argsFromContext: {
|
||||
nickname: "$request.nickname",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "도구 선실행",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "검사 결과 형식이 맞아요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "검사 결과 형식이 맞아요." } }] },
|
||||
message: { role: "assistant" as const, content: "검사 결과 형식이 맞아요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
const tool: Tool<{ nickname: string }, { valid: boolean }> = {
|
||||
name: "validate-nickname",
|
||||
description: "닉네임 검사",
|
||||
argsSchema: z.object({
|
||||
nickname: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
valid: z.boolean(),
|
||||
}),
|
||||
execute: (args) => Promise.resolve({ valid: args.nickname.includes("/") }),
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ responseMode: "llm-with-tools", tier: 1 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "닉네임 봐줘" },
|
||||
],
|
||||
{
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
},
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toBe("검사 결과 형식이 맞아요.");
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[TOOL_PLAN_RESULTS]");
|
||||
expect(calls[1]?.messages[0]?.content).toContain('"valid":true');
|
||||
expect(calls[1]?.options?.tools).toBeUndefined();
|
||||
});
|
||||
|
||||
it("injects requested SKILL.md resources before responder when enabled", async () => {
|
||||
process.env["AGENT_SKILL_MD_MODE"] = "true";
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "answer-direct",
|
||||
targetPhase: "photo-pending",
|
||||
mustInclude: ["하트"],
|
||||
mustAvoid: [],
|
||||
suggestedTools: [],
|
||||
requestResources: ["heart-rule"],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "리소스 참고",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "커버 보기에서 하트를 빨간색으로 바꿔주세요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: {
|
||||
choices: [
|
||||
{ message: { content: "커버 보기에서 하트를 빨간색으로 바꿔주세요." } },
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant" as const,
|
||||
content: "커버 보기에서 하트를 빨간색으로 바꿔주세요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 1 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "하트 어디서 해요?" },
|
||||
],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toContain("커버 보기");
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[RESOURCE:heart-rule]");
|
||||
});
|
||||
|
||||
|
||||
|
||||
it("runs critic after responder for tier 2 and applies rewrite", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { getContext, getRequestLogger } from "@obs/context";
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import {
|
||||
getAgentCriticMaxIterations,
|
||||
getAgentProgrammaticToolCalling,
|
||||
getLlmTemperature,
|
||||
getPlannerMaxTokens,
|
||||
getTier0EstimatedCallMs,
|
||||
@@ -16,14 +19,17 @@ import {
|
||||
getTier2MaxTokens,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { buildSkillResourceAddendum } from "../prompts/builder.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
import type { PlannerOutput } from "./planner-schema.js";
|
||||
import { PlannerOutputSchema } from "./planner-schema.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
import { runToolPlan, type ToolPlanResult } from "./tool-plan-runner.js";
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
|
||||
export interface OrchestratorRunOptions {
|
||||
maxTokens?: number | undefined;
|
||||
request?: AgentRequest | undefined;
|
||||
temperature?: number | undefined;
|
||||
toolContext: ToolContext;
|
||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||
@@ -100,6 +106,7 @@ function getDefaultPlan(derived: DerivedState): PlannerOutput {
|
||||
targetPhase: derived.phase.kind,
|
||||
mustInclude: [],
|
||||
mustAvoid: [],
|
||||
requestResources: [],
|
||||
suggestedTools:
|
||||
derived.responseMode === "llm-with-tools" ? ["validate-nickname"] : [],
|
||||
tone: derived.allowShortResponse ? "concise" : "friendly",
|
||||
@@ -154,6 +161,22 @@ function buildPlannerMessages(
|
||||
suggestedTools: [
|
||||
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context | verify-discord-nickname",
|
||||
],
|
||||
requestResources: [
|
||||
"nickname-format | heart-rule | discord-format | age-policy",
|
||||
],
|
||||
toolPlan: {
|
||||
composeWith: "all-then-llm",
|
||||
steps: [
|
||||
{
|
||||
id: "step-id",
|
||||
tool: "validate-nickname",
|
||||
argsLiteral: { nickname: "value" },
|
||||
argsFromContext: { chatroomId: "$request.chatroomId" },
|
||||
dependsOn: [],
|
||||
on: { error: "fallback-llm" },
|
||||
},
|
||||
],
|
||||
},
|
||||
tone: "friendly | concise | encouraging | neutral | apologetic",
|
||||
targetLengthChars: 150,
|
||||
rationale: "short reason",
|
||||
@@ -184,6 +207,23 @@ function buildResponderAddendum(plan: PlannerOutput): string {
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function buildToolPlanAddendum(result: ToolPlanResult): string {
|
||||
return [
|
||||
"[TOOL_PLAN_RESULTS]",
|
||||
JSON.stringify({
|
||||
durationMs: result.durationMs,
|
||||
failures: result.failures.map((failure) => ({
|
||||
kind: failure.kind,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
toolName: failure.toolName,
|
||||
})),
|
||||
results: result.results,
|
||||
}),
|
||||
"위 toolPlan 결과를 사실 정보로 사용하고, 실패한 도구 결과는 추측으로 메우지 마세요.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function pickToolsForPlan(
|
||||
plan: PlannerOutput,
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
@@ -237,6 +277,7 @@ export class OnboardingOrchestrator {
|
||||
messages,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
routingDecision: derived.routingDecision,
|
||||
tier: derived.tier,
|
||||
temperature: options.temperature ?? getLlmTemperature(),
|
||||
toolContext: options.toolContext,
|
||||
@@ -249,6 +290,7 @@ export class OnboardingOrchestrator {
|
||||
maxTokens,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
routingDecision: derived.routingDecision,
|
||||
temperature: options.temperature ?? getLlmTemperature(),
|
||||
tier: derived.tier,
|
||||
});
|
||||
@@ -267,8 +309,10 @@ export class OnboardingOrchestrator {
|
||||
approach: plan.approach,
|
||||
mustAvoid: plan.mustAvoid,
|
||||
mustInclude: plan.mustInclude,
|
||||
requestResources: plan.requestResources,
|
||||
suggestedTools: plan.suggestedTools,
|
||||
targetLengthChars: plan.targetLengthChars,
|
||||
toolPlanStepCount: plan.toolPlan?.steps.length,
|
||||
});
|
||||
return plan;
|
||||
};
|
||||
@@ -279,6 +323,7 @@ export class OnboardingOrchestrator {
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
routingDecision: derived.routingDecision,
|
||||
temperature: 0,
|
||||
tier: 1,
|
||||
});
|
||||
@@ -313,6 +358,7 @@ export class OnboardingOrchestrator {
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
routingDecision: derived.routingDecision,
|
||||
temperature: 0,
|
||||
tier: 1,
|
||||
});
|
||||
@@ -372,11 +418,32 @@ export class OnboardingOrchestrator {
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const forceTools = derived.responseMode === "llm-with-tools";
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
|
||||
let finalResponderMessages = responderMessages;
|
||||
let responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
|
||||
const resourceAddendum = buildSkillResourceAddendum(plan.requestResources);
|
||||
if (resourceAddendum) {
|
||||
finalResponderMessages = injectSystemAddendum(
|
||||
finalResponderMessages,
|
||||
resourceAddendum,
|
||||
);
|
||||
}
|
||||
const toolPlanResult = await this.runProgrammaticToolPlan(
|
||||
plan,
|
||||
finalResponderMessages,
|
||||
options,
|
||||
);
|
||||
|
||||
if (toolPlanResult && !toolPlanResult.fallbackToLlm) {
|
||||
finalResponderMessages = injectSystemAddendum(
|
||||
finalResponderMessages,
|
||||
buildToolPlanAddendum(toolPlanResult),
|
||||
);
|
||||
responderTools = [];
|
||||
}
|
||||
|
||||
return this.runDirect(
|
||||
derived,
|
||||
responderMessages,
|
||||
finalResponderMessages,
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier1MaxTokens(),
|
||||
@@ -395,11 +462,32 @@ export class OnboardingOrchestrator {
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const forceTools = derived.responseMode === "llm-with-tools";
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
|
||||
let finalResponderMessages = responderMessages;
|
||||
let responderTools = pickToolsForPlan(plan, options.tools ?? [], forceTools);
|
||||
const resourceAddendum = buildSkillResourceAddendum(plan.requestResources);
|
||||
if (resourceAddendum) {
|
||||
finalResponderMessages = injectSystemAddendum(
|
||||
finalResponderMessages,
|
||||
resourceAddendum,
|
||||
);
|
||||
}
|
||||
const toolPlanResult = await this.runProgrammaticToolPlan(
|
||||
plan,
|
||||
finalResponderMessages,
|
||||
options,
|
||||
);
|
||||
|
||||
if (toolPlanResult && !toolPlanResult.fallbackToLlm) {
|
||||
finalResponderMessages = injectSystemAddendum(
|
||||
finalResponderMessages,
|
||||
buildToolPlanAddendum(toolPlanResult),
|
||||
);
|
||||
responderTools = [];
|
||||
}
|
||||
|
||||
const draft = await this.runDirect(
|
||||
derived,
|
||||
responderMessages,
|
||||
finalResponderMessages,
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier2MaxTokens(),
|
||||
@@ -410,12 +498,46 @@ export class OnboardingOrchestrator {
|
||||
return null;
|
||||
}
|
||||
|
||||
const critique = await this.critic.critique(draft, derived, responderMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
});
|
||||
const maxIterations = getAgentCriticMaxIterations();
|
||||
const critique =
|
||||
maxIterations > 1
|
||||
? await this.critic.critiqueIterative(
|
||||
draft,
|
||||
derived,
|
||||
finalResponderMessages,
|
||||
{
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
maxIterations,
|
||||
},
|
||||
)
|
||||
: await this.critic.critique(draft, derived, finalResponderMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
});
|
||||
return critique.text;
|
||||
}
|
||||
|
||||
private async runProgrammaticToolPlan(
|
||||
plan: PlannerOutput,
|
||||
_messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<ToolPlanResult | null> {
|
||||
if (
|
||||
!getAgentProgrammaticToolCalling() ||
|
||||
!plan.toolPlan ||
|
||||
!options.request ||
|
||||
!options.tools?.length
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return runToolPlan(
|
||||
plan.toolPlan,
|
||||
options.tools,
|
||||
options.toolContext,
|
||||
options.request,
|
||||
);
|
||||
}
|
||||
|
||||
async run(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
|
||||
@@ -38,15 +38,47 @@ export const PlannerToneSchema = z.enum([
|
||||
"apologetic",
|
||||
]);
|
||||
|
||||
export const ToolPlanStepSchema = z.object({
|
||||
id: z.string().trim().min(1).max(40),
|
||||
tool: PlannerSuggestedToolSchema,
|
||||
argsFromContext: z.record(z.string(), z.string()).optional(),
|
||||
argsLiteral: z.record(z.string(), z.unknown()).optional(),
|
||||
dependsOn: z.array(z.string().trim().min(1)).max(3).optional(),
|
||||
on: z
|
||||
.object({
|
||||
error: z.enum(["abort", "fallback-llm"]).optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
export const ToolPlanSchema = z.object({
|
||||
steps: z.array(ToolPlanStepSchema).max(4),
|
||||
composeWith: z.enum(["all-then-llm", "first-then-llm"]),
|
||||
});
|
||||
|
||||
export const PlannerOutputSchema = z.object({
|
||||
approach: PlannerApproachSchema,
|
||||
targetPhase: PlannerTargetPhaseSchema.optional(),
|
||||
mustInclude: z.array(z.string().min(1).max(80)).max(5).default([]),
|
||||
mustAvoid: z.array(z.string().min(1).max(80)).max(5).default([]),
|
||||
suggestedTools: z.array(PlannerSuggestedToolSchema).max(3).default([]),
|
||||
requestResources: z
|
||||
.array(
|
||||
z.enum([
|
||||
"nickname-format",
|
||||
"heart-rule",
|
||||
"discord-format",
|
||||
"age-policy",
|
||||
]),
|
||||
)
|
||||
.max(4)
|
||||
.default([]),
|
||||
toolPlan: ToolPlanSchema.optional(),
|
||||
tone: PlannerToneSchema.default("friendly"),
|
||||
targetLengthChars: z.number().int().min(20).max(500).default(150),
|
||||
rationale: z.string().max(200),
|
||||
});
|
||||
|
||||
export type PlannerOutput = z.infer<typeof PlannerOutputSchema>;
|
||||
export type ToolPlan = z.infer<typeof ToolPlanSchema>;
|
||||
export type ToolPlanStep = z.infer<typeof ToolPlanStepSchema>;
|
||||
@@ -0,0 +1,394 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
Intent,
|
||||
OnboardingPhase,
|
||||
} from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession } from "../types.js";
|
||||
import {
|
||||
decideHybridRouting,
|
||||
decidePreGenerationRouting,
|
||||
extractRouterFeatures,
|
||||
loadLearnedRouterModel,
|
||||
predictRoutingDecisionKind,
|
||||
saveLearnedRouterModel,
|
||||
trainLearnedRouterModel,
|
||||
} from "./router.js";
|
||||
|
||||
function intent(kind: Intent["kind"]): Intent {
|
||||
return { kind };
|
||||
}
|
||||
|
||||
function request(overrides?: Partial<AgentRequest>): AgentRequest {
|
||||
return {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "도와주세요",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function session(overrides?: Partial<OnboardingSession>): OnboardingSession {
|
||||
return {
|
||||
chatroomId: "room-1",
|
||||
deadlineAt: "2026-05-02T00:00:00.000Z",
|
||||
joinedAt: "2026-05-01T00:00:00.000Z",
|
||||
kakaotalkNickname: "프리리/95/여/D",
|
||||
kakaotalkNicknameAliases: [],
|
||||
kakaotalkNicknameVerified: false,
|
||||
lastEventAt: "2026-05-01T00:00:00.000Z",
|
||||
nicknameVerified: false,
|
||||
photoVerified: false,
|
||||
status: "pending",
|
||||
userId: "user-1",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function derived(
|
||||
phase: OnboardingPhase,
|
||||
nextIntent: Intent,
|
||||
overrides?: Partial<DerivedState>,
|
||||
): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: nextIntent,
|
||||
phase,
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 256,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function classified(
|
||||
kind: Intent["kind"],
|
||||
confidence: number,
|
||||
): ClassifiedIntent {
|
||||
return {
|
||||
confidence,
|
||||
intent: intent(kind),
|
||||
source: "regex",
|
||||
};
|
||||
}
|
||||
|
||||
describe("decidePreGenerationRouting", () => {
|
||||
it.each([
|
||||
{
|
||||
confidence: 0.91,
|
||||
expected: { kind: "abstain", reason: "off-topic-strong" },
|
||||
intentKind: "off-topic-to-other" as const,
|
||||
name: "abstains on high-confidence off-topic",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.9,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "off-topic-to-other" as const,
|
||||
name: "does not abstain at the off-topic confidence boundary",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.3,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "off-topic-to-other" as const,
|
||||
name: "keeps low-confidence off-topic on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.96,
|
||||
expected: { kind: "template-only", templateKey: "greeting" },
|
||||
intentKind: "greeting" as const,
|
||||
name: "uses a greeting template",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.95,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "greeting" as const,
|
||||
name: "does not template greeting at the confidence boundary",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.99,
|
||||
expected: { kind: "template-only", templateKey: "greeting" },
|
||||
intentKind: "greeting" as const,
|
||||
name: "uses greeting template from a pending phase",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.91,
|
||||
expected: { kind: "template-only", templateKey: "acknowledgement" },
|
||||
intentKind: "acknowledgement" as const,
|
||||
name: "uses an acknowledgement template",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.9,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "acknowledgement" as const,
|
||||
name: "does not template acknowledgement at the confidence boundary",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "template-only", templateKey: "completed" },
|
||||
intentKind: "help-how" as const,
|
||||
name: "keeps completed sessions on template path",
|
||||
phase: { kind: "completed" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "status-question" as const,
|
||||
name: "lets completed status questions through",
|
||||
phase: { kind: "completed" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "template-only", templateKey: "completed" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "uses completed template for completed phase simple questions",
|
||||
phase: { kind: "completed" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "nickname-question" as const,
|
||||
name: "routes nickname help through tools",
|
||||
phase: { kind: "nickname-pending", current: "프리리" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "nickname-question" as const,
|
||||
name: "routes nickname-rejected phase through tools",
|
||||
phase: {
|
||||
kind: "nickname-rejected",
|
||||
current: "프리리/95/여D",
|
||||
issues: [],
|
||||
} as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "photo-question" as const,
|
||||
name: "routes photo help through tools",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "photo-question" as const,
|
||||
name: "routes photo help through tools from both-pending",
|
||||
phase: { kind: "both-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "help-how" as const,
|
||||
name: "routes general step-one help through tools",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "help-where" as const,
|
||||
name: "routes where-help through tools",
|
||||
phase: { kind: "both-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "help-how" as const,
|
||||
name: "keeps later phase general help on tier0",
|
||||
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "discord-verify-question" as const,
|
||||
name: "routes discord verification through tools",
|
||||
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-with-tools" },
|
||||
intentKind: "discord-verify-question" as const,
|
||||
name: "routes discord verification through tools from fresh phase",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-direct", skipPlanner: false },
|
||||
intentKind: "status-question" as const,
|
||||
name: "keeps deadline-near on tier1",
|
||||
phase: { kind: "deadline-near", remaining: ["photo"] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier1-direct", skipPlanner: false },
|
||||
intentKind: "age-question" as const,
|
||||
name: "routes deadline-near simple questions through tier1",
|
||||
phase: { kind: "deadline-near", remaining: ["nickname"] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "defaults simple questions to tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "keeps age questions on tier0 during photo-pending",
|
||||
phase: { kind: "photo-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "status-question" as const,
|
||||
name: "keeps ordinary status questions on tier0",
|
||||
phase: { kind: "both-pending" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "discord-question" as const,
|
||||
name: "keeps generic discord questions on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "unknown" as const,
|
||||
name: "keeps unknown intent on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "chitchat" as const,
|
||||
name: "keeps chitchat on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "age-question" as const,
|
||||
name: "keeps simple questions on tier0 during photo rejection",
|
||||
phase: { kind: "photo-rejected", reason: "unclear" } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.8,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "nickname-question" as const,
|
||||
name: "keeps nickname question on tier0 after step one",
|
||||
phase: { kind: "step1-completed", remaining: [] } as OnboardingPhase,
|
||||
},
|
||||
{
|
||||
confidence: 0.89,
|
||||
expected: { kind: "tier0-only" },
|
||||
intentKind: "acknowledgement" as const,
|
||||
name: "keeps lower-confidence acknowledgement on tier0",
|
||||
phase: { kind: "fresh" } as OnboardingPhase,
|
||||
},
|
||||
])("$name", ({ confidence, expected, intentKind, phase }) => {
|
||||
const nextIntent = intent(intentKind);
|
||||
expect(
|
||||
decidePreGenerationRouting({
|
||||
classified: classified(intentKind, confidence),
|
||||
derived: derived(phase, nextIntent),
|
||||
request: request(),
|
||||
session: session(),
|
||||
}),
|
||||
).toEqual(expected);
|
||||
});
|
||||
|
||||
it("trains, saves, loads, and applies a learned router model", () => {
|
||||
const lowConfidenceInput = {
|
||||
classified: classified("age-question", 0.25),
|
||||
derived: derived({ kind: "fresh" }, intent("age-question")),
|
||||
request: request({ userMessage: "나이 제한이 있나요?" }),
|
||||
session: session(),
|
||||
};
|
||||
const highConfidenceInput = {
|
||||
classified: classified("age-question", 0.95),
|
||||
derived: derived({ kind: "fresh" }, intent("age-question")),
|
||||
request: request({ userMessage: "나이 제한이 있나요?" }),
|
||||
session: session(),
|
||||
};
|
||||
const model = trainLearnedRouterModel(
|
||||
[
|
||||
{
|
||||
features: extractRouterFeatures(lowConfidenceInput),
|
||||
label: "tier1-direct",
|
||||
},
|
||||
{
|
||||
features: extractRouterFeatures(highConfidenceInput),
|
||||
label: "tier0-only",
|
||||
},
|
||||
],
|
||||
{
|
||||
epochs: 200,
|
||||
learningRate: 0.5,
|
||||
trainedAt: "2026-05-01T00:00:00.000Z",
|
||||
},
|
||||
);
|
||||
const path = join(mkdtempSync(join(tmpdir(), "router-")), "weights.json");
|
||||
|
||||
saveLearnedRouterModel(path, model);
|
||||
const loaded = loadLearnedRouterModel(path);
|
||||
|
||||
expect(loaded?.trainedAt).toBe("2026-05-01T00:00:00.000Z");
|
||||
expect(
|
||||
predictRoutingDecisionKind(model, extractRouterFeatures(lowConfidenceInput)),
|
||||
).toBe("tier1-direct");
|
||||
expect(
|
||||
predictRoutingDecisionKind(model, extractRouterFeatures(highConfidenceInput)),
|
||||
).toBe("tier0-only");
|
||||
expect(
|
||||
decideHybridRouting(lowConfidenceInput, { learnedModel: model }),
|
||||
).toEqual({
|
||||
kind: "tier1-direct",
|
||||
skipPlanner: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps static abstain and template guards in hybrid routing", () => {
|
||||
const offTopicInput = {
|
||||
classified: classified("off-topic-to-other", 0.99),
|
||||
derived: derived({ kind: "fresh" }, intent("off-topic-to-other")),
|
||||
request: request({ userMessage: "다른 주제로 얘기하자" }),
|
||||
session: session(),
|
||||
};
|
||||
const model = trainLearnedRouterModel(
|
||||
[
|
||||
{
|
||||
features: extractRouterFeatures(offTopicInput),
|
||||
label: "tier2-full",
|
||||
},
|
||||
],
|
||||
{ epochs: 10 },
|
||||
);
|
||||
|
||||
expect(decideHybridRouting(offTopicInput, { learnedModel: model })).toEqual({
|
||||
kind: "abstain",
|
||||
reason: "off-topic-strong",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,379 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
Intent,
|
||||
OnboardingPhase,
|
||||
} from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession } from "../types.js";
|
||||
|
||||
export type RoutingDecision =
|
||||
| { kind: "abstain"; reason: string }
|
||||
| { kind: "template-only"; templateKey: string }
|
||||
| { kind: "tier0-only" }
|
||||
| { kind: "tier1-direct"; skipPlanner: boolean }
|
||||
| { kind: "tier1-with-tools" }
|
||||
| { kind: "tier2-full" };
|
||||
|
||||
export const ROUTING_DECISION_KINDS = [
|
||||
"abstain",
|
||||
"template-only",
|
||||
"tier0-only",
|
||||
"tier1-direct",
|
||||
"tier1-with-tools",
|
||||
"tier2-full",
|
||||
] as const;
|
||||
|
||||
export type RoutingDecisionKind = (typeof ROUTING_DECISION_KINDS)[number];
|
||||
|
||||
export interface RouterFeatureVector {
|
||||
classifyConfidence: number;
|
||||
classifySource: 0 | 1;
|
||||
intentKind: string;
|
||||
messageLength: number;
|
||||
phaseKind: string;
|
||||
responseModeTools: 0 | 1;
|
||||
sessionCompleted: 0 | 1;
|
||||
tier: 0 | 1 | 2;
|
||||
trigger: string;
|
||||
}
|
||||
|
||||
export interface LearnedRouterModel {
|
||||
decisionKinds: RoutingDecisionKind[];
|
||||
featureNames: string[];
|
||||
trainedAt: string;
|
||||
version: 1;
|
||||
weights: number[][];
|
||||
}
|
||||
|
||||
export interface RouterTrainingSample {
|
||||
features: RouterFeatureVector;
|
||||
label: RoutingDecisionKind;
|
||||
}
|
||||
|
||||
export interface RouterInput {
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState;
|
||||
request: AgentRequest;
|
||||
session: OnboardingSession | null;
|
||||
}
|
||||
|
||||
export const ROUTER_FEATURE_NAMES = [
|
||||
"bias",
|
||||
"classifyConfidence",
|
||||
"classifySource",
|
||||
"intentHash",
|
||||
"phaseHash",
|
||||
"triggerHash",
|
||||
"messageLength",
|
||||
"tier",
|
||||
"responseModeTools",
|
||||
"sessionCompleted",
|
||||
] as const;
|
||||
|
||||
const modelCache = new Map<string, LearnedRouterModel | null>();
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(6));
|
||||
}
|
||||
|
||||
function clamp01(value: number): number {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function stableBucket(value: string): number {
|
||||
const hash = createHash("sha256").update(value).digest();
|
||||
return round((hash[0] ?? 0) / 255);
|
||||
}
|
||||
|
||||
function dot(left: readonly number[], right: readonly number[]): number {
|
||||
return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
|
||||
}
|
||||
|
||||
function softmax(scores: readonly number[]): number[] {
|
||||
const max = Math.max(...scores);
|
||||
const exp = scores.map((score) => Math.exp(score - max));
|
||||
const sum = exp.reduce((total, value) => total + value, 0);
|
||||
return exp.map((value) => value / Math.max(1e-9, sum));
|
||||
}
|
||||
|
||||
function decisionKindFromDecision(decision: RoutingDecision): RoutingDecisionKind {
|
||||
return decision.kind;
|
||||
}
|
||||
|
||||
function decisionFromKind(kind: RoutingDecisionKind, input: RouterInput): RoutingDecision {
|
||||
switch (kind) {
|
||||
case "abstain":
|
||||
return {
|
||||
kind: "abstain",
|
||||
reason:
|
||||
input.classified.intent.kind === "off-topic-to-other"
|
||||
? "off-topic-strong"
|
||||
: "learned-router",
|
||||
};
|
||||
case "template-only":
|
||||
if (input.classified.intent.kind === "acknowledgement") {
|
||||
return { kind, templateKey: "acknowledgement" };
|
||||
}
|
||||
if (
|
||||
input.derived.phase.kind === "completed" ||
|
||||
input.request.trigger === "completed"
|
||||
) {
|
||||
return { kind, templateKey: "completed" };
|
||||
}
|
||||
return { kind, templateKey: "greeting" };
|
||||
case "tier1-direct":
|
||||
return { kind, skipPlanner: false };
|
||||
case "tier0-only":
|
||||
case "tier1-with-tools":
|
||||
case "tier2-full":
|
||||
return { kind };
|
||||
}
|
||||
}
|
||||
|
||||
export function extractRouterFeatures(input: RouterInput): RouterFeatureVector {
|
||||
return {
|
||||
classifyConfidence: clamp01(input.classified.confidence),
|
||||
classifySource: input.classified.source === "llm" ? 1 : 0,
|
||||
intentKind: input.classified.intent.kind,
|
||||
messageLength: input.request.userMessage?.trim().length ?? 0,
|
||||
phaseKind: input.derived.phase.kind,
|
||||
responseModeTools: input.derived.responseMode === "llm-with-tools" ? 1 : 0,
|
||||
sessionCompleted: input.session?.status === "completed" ? 1 : 0,
|
||||
tier: input.derived.tier,
|
||||
trigger: input.request.trigger,
|
||||
};
|
||||
}
|
||||
|
||||
export function encodeRouterFeatures(features: RouterFeatureVector): number[] {
|
||||
return [
|
||||
1,
|
||||
features.classifyConfidence,
|
||||
features.classifySource,
|
||||
stableBucket(features.intentKind),
|
||||
stableBucket(features.phaseKind),
|
||||
stableBucket(features.trigger),
|
||||
Math.min(500, features.messageLength) / 500,
|
||||
features.tier / 2,
|
||||
features.responseModeTools,
|
||||
features.sessionCompleted,
|
||||
];
|
||||
}
|
||||
|
||||
export function trainLearnedRouterModel(
|
||||
samples: RouterTrainingSample[],
|
||||
options?: {
|
||||
epochs?: number | undefined;
|
||||
learningRate?: number | undefined;
|
||||
l2?: number | undefined;
|
||||
trainedAt?: string | undefined;
|
||||
},
|
||||
): LearnedRouterModel {
|
||||
const weights = ROUTING_DECISION_KINDS.map(() =>
|
||||
ROUTER_FEATURE_NAMES.map(() => 0),
|
||||
);
|
||||
const epochs = options?.epochs ?? 200;
|
||||
const learningRate = options?.learningRate ?? 0.2;
|
||||
const l2 = options?.l2 ?? 0.001;
|
||||
|
||||
for (let epoch = 0; epoch < epochs; epoch++) {
|
||||
for (const sample of samples) {
|
||||
const encoded = encodeRouterFeatures(sample.features);
|
||||
const targetIndex = ROUTING_DECISION_KINDS.indexOf(sample.label);
|
||||
if (targetIndex < 0) {
|
||||
continue;
|
||||
}
|
||||
const probs = softmax(weights.map((row) => dot(row, encoded)));
|
||||
for (let decisionIndex = 0; decisionIndex < weights.length; decisionIndex++) {
|
||||
const row = weights[decisionIndex];
|
||||
if (!row) {
|
||||
continue;
|
||||
}
|
||||
const error = (decisionIndex === targetIndex ? 1 : 0) - (probs[decisionIndex] ?? 0);
|
||||
for (let featureIndex = 0; featureIndex < row.length; featureIndex++) {
|
||||
row[featureIndex] =
|
||||
(row[featureIndex] ?? 0) +
|
||||
learningRate *
|
||||
(error * (encoded[featureIndex] ?? 0) -
|
||||
l2 * (row[featureIndex] ?? 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
decisionKinds: [...ROUTING_DECISION_KINDS],
|
||||
featureNames: [...ROUTER_FEATURE_NAMES],
|
||||
trainedAt: options?.trainedAt ?? new Date().toISOString(),
|
||||
version: 1,
|
||||
weights: weights.map((row) => row.map(round)),
|
||||
};
|
||||
}
|
||||
|
||||
export function predictRoutingDecisionKind(
|
||||
model: LearnedRouterModel,
|
||||
features: RouterFeatureVector,
|
||||
): RoutingDecisionKind {
|
||||
const encoded = encodeRouterFeatures(features);
|
||||
const scores = model.weights.map((row) => dot(row, encoded));
|
||||
const bestIndex = scores.reduce(
|
||||
(best, score, index) => (score > (scores[best] ?? Number.NEGATIVE_INFINITY) ? index : best),
|
||||
0,
|
||||
);
|
||||
return model.decisionKinds[bestIndex] ?? "tier0-only";
|
||||
}
|
||||
|
||||
export function decideLearnedRouting(
|
||||
input: RouterInput,
|
||||
model: LearnedRouterModel,
|
||||
): RoutingDecision {
|
||||
return decisionFromKind(
|
||||
predictRoutingDecisionKind(model, extractRouterFeatures(input)),
|
||||
input,
|
||||
);
|
||||
}
|
||||
|
||||
export function loadLearnedRouterModel(path: string | undefined): LearnedRouterModel | null {
|
||||
if (!path?.trim()) {
|
||||
return null;
|
||||
}
|
||||
const cached = modelCache.get(path);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
if (!existsSync(path)) {
|
||||
modelCache.set(path, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as LearnedRouterModel;
|
||||
if (
|
||||
parsed.version !== 1 ||
|
||||
parsed.featureNames.length !== ROUTER_FEATURE_NAMES.length ||
|
||||
parsed.decisionKinds.length !== ROUTING_DECISION_KINDS.length ||
|
||||
parsed.weights.length !== ROUTING_DECISION_KINDS.length ||
|
||||
parsed.weights.some((row) => row.length !== ROUTER_FEATURE_NAMES.length)
|
||||
) {
|
||||
modelCache.set(path, null);
|
||||
return null;
|
||||
}
|
||||
modelCache.set(path, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
modelCache.set(path, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function saveLearnedRouterModel(path: string, model: LearnedRouterModel): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(model, null, 2)}\n`);
|
||||
modelCache.set(path, model);
|
||||
}
|
||||
|
||||
function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
|
||||
if (
|
||||
phase.kind !== "fresh" &&
|
||||
phase.kind !== "nickname-pending" &&
|
||||
phase.kind !== "photo-pending" &&
|
||||
phase.kind !== "both-pending"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
intent.kind === "nickname-question" ||
|
||||
intent.kind === "photo-question" ||
|
||||
intent.kind === "help-how" ||
|
||||
intent.kind === "help-where"
|
||||
);
|
||||
}
|
||||
|
||||
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
|
||||
const { classified, derived, request, session } = input;
|
||||
const { confidence, intent } = classified;
|
||||
const { phase } = derived;
|
||||
|
||||
if (intent.kind === "off-topic-to-other" && confidence > 0.9) {
|
||||
return {
|
||||
kind: "abstain",
|
||||
reason: "off-topic-strong",
|
||||
};
|
||||
}
|
||||
|
||||
if (intent.kind === "greeting" && confidence > 0.95) {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "greeting",
|
||||
};
|
||||
}
|
||||
|
||||
if (intent.kind === "acknowledgement" && confidence > 0.9) {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "acknowledgement",
|
||||
};
|
||||
}
|
||||
|
||||
if (phase.kind === "completed" && intent.kind !== "status-question") {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "completed",
|
||||
};
|
||||
}
|
||||
|
||||
if (request.trigger === "completed" || session?.status === "completed") {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "completed",
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
phase.kind === "nickname-rejected" ||
|
||||
intent.kind === "discord-verify-question" ||
|
||||
needsStepOneHelp(phase, intent)
|
||||
) {
|
||||
return {
|
||||
kind: "tier1-with-tools",
|
||||
};
|
||||
}
|
||||
|
||||
if (phase.kind === "deadline-near") {
|
||||
return {
|
||||
kind: "tier1-direct",
|
||||
skipPlanner: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "tier0-only",
|
||||
};
|
||||
}
|
||||
|
||||
export function decideHybridRouting(
|
||||
input: RouterInput,
|
||||
options?: {
|
||||
learnedModel?: LearnedRouterModel | null | undefined;
|
||||
},
|
||||
): RoutingDecision {
|
||||
if (!options?.learnedModel) {
|
||||
return decidePreGenerationRouting(input);
|
||||
}
|
||||
|
||||
const staticDecision = decidePreGenerationRouting(input);
|
||||
if (staticDecision.kind === "abstain" || staticDecision.kind === "template-only") {
|
||||
return staticDecision;
|
||||
}
|
||||
|
||||
return decideLearnedRouting(input, options.learnedModel);
|
||||
}
|
||||
|
||||
export function routerDecisionKind(decision: RoutingDecision): RoutingDecisionKind {
|
||||
return decisionKindFromDecision(decision);
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
loadRuntimeFlagManifest,
|
||||
parseRuntimeFlagManifest,
|
||||
resolveRuntimeFlags,
|
||||
type ResolvedRuntimeFlags,
|
||||
} from "./runtime-flag-manifest.js";
|
||||
|
||||
const baseFlags: ResolvedRuntimeFlags = {
|
||||
abstentionClarifyThreshold: 0.45,
|
||||
abstentionEnabled: false,
|
||||
abstentionThreshold: 0.35,
|
||||
criticMaxIterations: 2,
|
||||
programmaticToolCalling: false,
|
||||
routerMode: "static",
|
||||
toolLoopMaxHops: 3,
|
||||
userMemoryEnabled: false,
|
||||
};
|
||||
|
||||
describe("runtime flag manifests", () => {
|
||||
it("loads built-in runtime variants", () => {
|
||||
const manifest = loadRuntimeFlagManifest("abstention-on");
|
||||
|
||||
expect(manifest).toMatchObject({
|
||||
variantId: "abstention-on",
|
||||
flags: {
|
||||
abstentionEnabled: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("merges variant flags over base flags", () => {
|
||||
const manifest = loadRuntimeFlagManifest("toolplan-on");
|
||||
const resolved = resolveRuntimeFlags(baseFlags, manifest);
|
||||
|
||||
expect(resolved).toMatchObject({
|
||||
abstentionEnabled: false,
|
||||
programmaticToolCalling: true,
|
||||
toolLoopMaxHops: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects unknown flag keys", () => {
|
||||
const manifest = parseRuntimeFlagManifest({
|
||||
variantId: "bad",
|
||||
description: "bad manifest",
|
||||
flags: {
|
||||
shellCommand: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(manifest).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,186 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
export type RouterMode = "static" | "learned-v1" | "learned-v2";
|
||||
|
||||
export interface RuntimeFlagManifest {
|
||||
description: string;
|
||||
flags: Partial<RuntimeFlagSet>;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
export interface RuntimeFlagSet {
|
||||
abstentionClarifyThreshold: number;
|
||||
abstentionEnabled: boolean;
|
||||
abstentionThreshold: number;
|
||||
criticMaxIterations: number;
|
||||
programmaticToolCalling: boolean;
|
||||
routerMode: RouterMode;
|
||||
toolLoopMaxHops: number;
|
||||
userMemoryEnabled: boolean;
|
||||
}
|
||||
|
||||
export type ResolvedRuntimeFlags = RuntimeFlagSet;
|
||||
|
||||
const runtimeManifestCache = new Map<string, RuntimeFlagManifest | null>();
|
||||
const allowedFlagKeys = new Set<keyof RuntimeFlagSet>([
|
||||
"abstentionClarifyThreshold",
|
||||
"abstentionEnabled",
|
||||
"abstentionThreshold",
|
||||
"criticMaxIterations",
|
||||
"programmaticToolCalling",
|
||||
"routerMode",
|
||||
"toolLoopMaxHops",
|
||||
"userMemoryEnabled",
|
||||
]);
|
||||
|
||||
const builtInManifests: Record<string, RuntimeFlagManifest> = {
|
||||
"abstention-on": {
|
||||
variantId: "abstention-on",
|
||||
description: "Enable hybrid early abstention with default thresholds.",
|
||||
flags: {
|
||||
abstentionEnabled: true,
|
||||
},
|
||||
},
|
||||
"toolplan-on": {
|
||||
variantId: "toolplan-on",
|
||||
description: "Enable planner-declared programmatic tool plans.",
|
||||
flags: {
|
||||
programmaticToolCalling: true,
|
||||
toolLoopMaxHops: 3,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function isRouterMode(value: unknown): value is RouterMode {
|
||||
return (
|
||||
value === "static" ||
|
||||
value === "learned-v1" ||
|
||||
value === "learned-v2"
|
||||
);
|
||||
}
|
||||
|
||||
function parseFlagValue(
|
||||
key: keyof RuntimeFlagSet,
|
||||
value: unknown,
|
||||
): RuntimeFlagSet[keyof RuntimeFlagSet] | undefined {
|
||||
switch (key) {
|
||||
case "abstentionEnabled":
|
||||
case "programmaticToolCalling":
|
||||
case "userMemoryEnabled":
|
||||
return typeof value === "boolean" ? value : undefined;
|
||||
case "routerMode":
|
||||
return isRouterMode(value) ? value : undefined;
|
||||
case "abstentionClarifyThreshold":
|
||||
case "abstentionThreshold":
|
||||
return typeof value === "number" &&
|
||||
Number.isFinite(value) &&
|
||||
value >= 0 &&
|
||||
value <= 1
|
||||
? value
|
||||
: undefined;
|
||||
case "criticMaxIterations":
|
||||
case "toolLoopMaxHops":
|
||||
return typeof value === "number" && Number.isInteger(value) && value > 0
|
||||
? value
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseRuntimeFlagManifest(
|
||||
value: unknown,
|
||||
expectedVariantId?: string,
|
||||
): RuntimeFlagManifest | null {
|
||||
const record = asRecord(value);
|
||||
const variantId = record["variantId"];
|
||||
const description = record["description"];
|
||||
const rawFlags = asRecord(record["flags"]);
|
||||
|
||||
if (
|
||||
typeof variantId !== "string" ||
|
||||
!variantId.trim() ||
|
||||
(expectedVariantId && variantId !== expectedVariantId) ||
|
||||
typeof description !== "string" ||
|
||||
!description.trim()
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const flags: Partial<RuntimeFlagSet> = {};
|
||||
for (const [key, rawValue] of Object.entries(rawFlags)) {
|
||||
if (!allowedFlagKeys.has(key as keyof RuntimeFlagSet)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = parseFlagValue(key as keyof RuntimeFlagSet, rawValue);
|
||||
if (parsed === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Object.assign(flags, { [key]: parsed });
|
||||
}
|
||||
|
||||
return {
|
||||
variantId,
|
||||
description,
|
||||
flags,
|
||||
};
|
||||
}
|
||||
|
||||
export function loadRuntimeFlagManifest(
|
||||
variantId: string,
|
||||
): RuntimeFlagManifest | null {
|
||||
const normalized = variantId.trim();
|
||||
if (!normalized || normalized === "control") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const builtIn = builtInManifests[normalized];
|
||||
if (builtIn) {
|
||||
return builtIn;
|
||||
}
|
||||
|
||||
if (runtimeManifestCache.has(normalized)) {
|
||||
return runtimeManifestCache.get(normalized) ?? null;
|
||||
}
|
||||
|
||||
const path = resolve(
|
||||
process.cwd(),
|
||||
"src",
|
||||
"eval",
|
||||
"proposals",
|
||||
`${normalized}.runtime.json`,
|
||||
);
|
||||
if (!existsSync(path)) {
|
||||
runtimeManifestCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = parseRuntimeFlagManifest(
|
||||
JSON.parse(readFileSync(path, "utf8")) as unknown,
|
||||
normalized,
|
||||
);
|
||||
runtimeManifestCache.set(normalized, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
runtimeManifestCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveRuntimeFlags(
|
||||
base: ResolvedRuntimeFlags,
|
||||
variant?: RuntimeFlagManifest | null,
|
||||
): ResolvedRuntimeFlags {
|
||||
return {
|
||||
...base,
|
||||
...(variant?.flags ?? {}),
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,8 @@ import { z } from "zod";
|
||||
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
import type { Tool } from "@tools/tool";
|
||||
import { ToolRegistry } from "@tools/registry";
|
||||
import { createToolSearchTool } from "@tools/tool-search";
|
||||
|
||||
describe("runWithTools", () => {
|
||||
it("executes a tool call and returns the model's final answer", async () => {
|
||||
@@ -106,6 +108,81 @@ describe("runWithTools", () => {
|
||||
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
|
||||
});
|
||||
|
||||
it("expands searched tools for the next tool-call hop", async () => {
|
||||
const toolOptionNames: string[][] = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
_messages: Array<Record<string, unknown>>,
|
||||
options?: { tools?: Array<{ name: string }> },
|
||||
) => {
|
||||
toolOptionNames.push(options?.tools?.map((tool) => tool.name) ?? []);
|
||||
|
||||
if (toolOptionNames.length === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"search_tools","arguments":{"query":"nickname validation","topK":1}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (toolOptionNames.length === 2) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "닉네임 형식을 다시 맞춰주세요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "닉네임 형식을 다시 맞춰주세요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
const validateTool: Tool<{ nickname: string }, { valid: boolean }> = {
|
||||
argsSchema: z.object({
|
||||
nickname: z.string(),
|
||||
}),
|
||||
description: "닉네임 형식을 검사한다.",
|
||||
execute: () => Promise.resolve({ valid: false }),
|
||||
name: "validate-nickname",
|
||||
resultSchema: z.object({
|
||||
valid: z.boolean(),
|
||||
}),
|
||||
};
|
||||
const registry = new ToolRegistry().register(validateTool);
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "닉네임 확인해줘" }],
|
||||
toolContext: { toolRegistry: registry },
|
||||
tools: [createToolSearchTool(registry)],
|
||||
});
|
||||
|
||||
expect(result).toContain("닉네임");
|
||||
expect(toolOptionNames[0]).toEqual(["search_tools"]);
|
||||
expect(toolOptionNames[1]).toEqual(["search_tools", "validate-nickname"]);
|
||||
});
|
||||
|
||||
it("preserves native tool call responses for follow-up turns", async () => {
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
|
||||
@@ -216,4 +293,291 @@ describe("runWithTools", () => {
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns permanent tool failures to the model instead of throwing", async () => {
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
seenMessages.push(messages);
|
||||
|
||||
if (seenMessages.length === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"explode","arguments":{"value":"x"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "지금은 확인이 어려워서 기본 안내만 드릴게요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ value: string }, { ok: boolean }> = {
|
||||
name: "explode",
|
||||
description: "항상 실패",
|
||||
argsSchema: z.object({
|
||||
value: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
}),
|
||||
execute: () => Promise.reject(new Error("boom")),
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("기본 안내");
|
||||
const toolMessage = seenMessages[1]?.find(
|
||||
(message) => message.role === "tool",
|
||||
);
|
||||
expect(JSON.parse(String(toolMessage?.content))).toMatchObject({
|
||||
errorKind: "permanent",
|
||||
});
|
||||
});
|
||||
|
||||
it("allows one self-correction hop when tool arguments are invalid", async () => {
|
||||
let executeCount = 0;
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
seenMessages.push(messages);
|
||||
|
||||
if (seenMessages.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: '<tool_call>{"name":"needs-name","arguments":{}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (seenMessages.length === 2) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"needs-name","arguments":{"name":"프리리"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "프리리님 기준으로 확인했어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "프리리님 기준으로 확인했어요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ name: string }, { ok: boolean }> = {
|
||||
name: "needs-name",
|
||||
description: "이름 필요",
|
||||
argsSchema: z.object({
|
||||
name: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
}),
|
||||
execute: () => {
|
||||
executeCount += 1;
|
||||
return Promise.resolve({ ok: true });
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("프리리님");
|
||||
expect(executeCount).toBe(1);
|
||||
expect(JSON.stringify(seenMessages[1])).toContain("schema");
|
||||
});
|
||||
|
||||
it("retries transient tool errors once before continuing", async () => {
|
||||
let executeCount = 0;
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
if (messages.some((message) => message.role === "tool")) {
|
||||
return Promise.resolve({
|
||||
content: "재시도 후 확인됐어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "재시도 후 확인됐어요.",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: '<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"flaky","arguments":{"value":"x"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ value: string }, { ok: boolean }> = {
|
||||
name: "flaky",
|
||||
description: "일시 실패",
|
||||
argsSchema: z.object({
|
||||
value: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
}),
|
||||
execute: () => {
|
||||
executeCount += 1;
|
||||
if (executeCount === 1) {
|
||||
const error = new Error("timeout while calling dependency");
|
||||
error.name = "AbortError";
|
||||
return Promise.reject(error);
|
||||
}
|
||||
|
||||
return Promise.resolve({ ok: true });
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toBe("재시도 후 확인됐어요.");
|
||||
expect(executeCount).toBe(2);
|
||||
});
|
||||
|
||||
it("grants one extra hop for precondition failures", async () => {
|
||||
let executeCount = 0;
|
||||
const llmClientStub = {
|
||||
chatCompletion: () => {
|
||||
if (executeCount === 0) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"first"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (executeCount === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"needs-session","arguments":{"value":"second"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "추가 확인까지 마쳤어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: { choices: [{ message: { content: "" } }] },
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "추가 확인까지 마쳤어요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<
|
||||
{ value: string },
|
||||
{ ok: boolean; errorKind?: string | undefined; message?: string | undefined }
|
||||
> = {
|
||||
name: "needs-session",
|
||||
description: "세션 필요",
|
||||
argsSchema: z.object({
|
||||
value: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
ok: z.boolean(),
|
||||
errorKind: z.string().optional(),
|
||||
message: z.string().optional(),
|
||||
}),
|
||||
execute: () => {
|
||||
executeCount += 1;
|
||||
if (executeCount === 1) {
|
||||
return Promise.resolve({
|
||||
ok: false,
|
||||
errorKind: "precondition-failed",
|
||||
message: "세션이 없어요.",
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({ ok: true });
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
maxHops: 1,
|
||||
messages: [{ role: "user", content: "확인해줘" }],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("추가 확인");
|
||||
expect(executeCount).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,13 @@ import { getRequestLogger } from "@obs/context";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { parseToolCalls } from "@tools/parser";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import type {
|
||||
Tool,
|
||||
ToolContext,
|
||||
ToolErrorKind,
|
||||
ToolExecutionFailure,
|
||||
} from "@tools/tool";
|
||||
import { getAgentToolLoopMaxHops, getAgentToolRetryMax } from "../../../config.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface ToolLoopOptions {
|
||||
@@ -14,6 +20,7 @@ export interface ToolLoopOptions {
|
||||
messages: ChatMessage[];
|
||||
modelOverride?: string | undefined;
|
||||
role?: ChatCompletionOptions["role"];
|
||||
routingDecision?: ChatCompletionOptions["routingDecision"];
|
||||
signal?: AbortSignal | undefined;
|
||||
temperature?: number | undefined;
|
||||
tier?: ChatCompletionOptions["tier"];
|
||||
@@ -25,20 +32,25 @@ export async function runWithTools(
|
||||
options: ToolLoopOptions,
|
||||
): Promise<string | null> {
|
||||
const logger = getRequestLogger();
|
||||
const maxHops = options.maxHops ?? 2;
|
||||
const maxHops = options.maxHops ?? getAgentToolLoopMaxHops();
|
||||
const maxRetries = getAgentToolRetryMax();
|
||||
const messages = options.messages.map((message) => ({ ...message }));
|
||||
const activeTools = [...options.tools];
|
||||
let allowNativeTools = true;
|
||||
let preconditionBonusHops = 0;
|
||||
let selfCorrectionHops = 0;
|
||||
|
||||
for (let hop = 0; hop <= maxHops; hop++) {
|
||||
for (let hop = 0; hop <= maxHops + preconditionBonusHops; hop++) {
|
||||
const response = await options.llmClient.chatCompletion(messages, {
|
||||
deadlineMs: options.deadlineMs,
|
||||
maxTokens: options.maxTokens,
|
||||
modelOverride: options.modelOverride,
|
||||
role: options.role,
|
||||
routingDecision: options.routingDecision,
|
||||
signal: options.signal,
|
||||
temperature: options.temperature,
|
||||
tier: options.tier,
|
||||
tools: allowNativeTools ? options.tools : undefined,
|
||||
tools: allowNativeTools ? activeTools : undefined,
|
||||
});
|
||||
|
||||
if (!response) {
|
||||
@@ -80,7 +92,7 @@ export async function runWithTools(
|
||||
"[ToolLoop] Parsed tool calls",
|
||||
);
|
||||
|
||||
if (hop === maxHops) {
|
||||
if (hop === maxHops + preconditionBonusHops) {
|
||||
messages.push(response.message);
|
||||
messages.push({
|
||||
role: "user",
|
||||
@@ -94,9 +106,16 @@ export async function runWithTools(
|
||||
messages.push(response.message);
|
||||
|
||||
for (const call of toolCalls) {
|
||||
const tool = options.tools.find((candidate) => candidate.name === call.name);
|
||||
const tool = activeTools.find((candidate) => candidate.name === call.name);
|
||||
if (!tool) {
|
||||
throw new Error(`Tool not registered for call: ${call.name}`);
|
||||
const failure = makeToolFailure({
|
||||
kind: "permanent",
|
||||
message: `Tool not registered for call: ${call.name}`,
|
||||
retryable: false,
|
||||
toolName: call.name,
|
||||
});
|
||||
pushToolFailureMessage(messages, call, failure);
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
@@ -110,30 +129,75 @@ export async function runWithTools(
|
||||
|
||||
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
||||
let result: unknown;
|
||||
let retried = false;
|
||||
try {
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
result = await withToolSpan(
|
||||
tool.name,
|
||||
result = await executeToolWithRetry(
|
||||
() =>
|
||||
withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
),
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
maxRetries,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
).then((execution) => {
|
||||
retried = execution.retried;
|
||||
return execution.result;
|
||||
});
|
||||
const preconditionFailure = getPreconditionFailure(tool.name, result);
|
||||
if (preconditionFailure) {
|
||||
if (
|
||||
preconditionFailure.kind === "precondition-failed" &&
|
||||
preconditionBonusHops < 1
|
||||
) {
|
||||
preconditionBonusHops += 1;
|
||||
}
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
errorKind: preconditionFailure.kind,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
retried,
|
||||
resultSummary: preconditionFailure.message,
|
||||
});
|
||||
pushToolFailureMessage(messages, call, preconditionFailure, result);
|
||||
continue;
|
||||
}
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
name: tool.name,
|
||||
ok: true,
|
||||
retried,
|
||||
resultSummary: summarizeToolResult(result),
|
||||
});
|
||||
} catch (error) {
|
||||
const failure = classifyToolError(tool.name, error);
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
errorKind: failure.kind,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: error instanceof Error ? error.message : String(error),
|
||||
retried: failure.kind === "transient" && maxRetries > 0,
|
||||
resultSummary: failure.message,
|
||||
});
|
||||
throw error;
|
||||
|
||||
pushToolFailureMessage(messages, call, failure);
|
||||
if (failure.kind === "args-invalid" && selfCorrectionHops < 1) {
|
||||
selfCorrectionHops += 1;
|
||||
messages.push({
|
||||
role: "user",
|
||||
content:
|
||||
`이전 ${tool.name} 도구 호출의 arguments가 schema와 맞지 않았어요. ` +
|
||||
"도구 arguments를 고쳐서 한 번만 다시 호출하거나, 도구 없이 최종 답변하세요.",
|
||||
});
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
@@ -145,6 +209,7 @@ export async function runWithTools(
|
||||
"[ToolLoop] Tool call completed",
|
||||
);
|
||||
|
||||
expandActiveToolsFromSearchResult(activeTools, options.toolContext, result);
|
||||
messages.push({
|
||||
role: "tool",
|
||||
name: tool.name,
|
||||
@@ -157,6 +222,182 @@ export async function runWithTools(
|
||||
return null;
|
||||
}
|
||||
|
||||
function expandActiveToolsFromSearchResult(
|
||||
activeTools: Tool<unknown, unknown>[],
|
||||
toolContext: ToolContext,
|
||||
result: unknown,
|
||||
): void {
|
||||
const registry = toolContext.toolRegistry;
|
||||
const tools = asRecord(result)["tools"];
|
||||
if (!registry || !Array.isArray(tools)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const existing = new Set(activeTools.map((tool) => tool.name));
|
||||
for (const item of tools) {
|
||||
const name = asRecord(item)["name"];
|
||||
if (typeof name !== "string" || existing.has(name) || !registry.has(name)) {
|
||||
continue;
|
||||
}
|
||||
const tool = registry.get(name);
|
||||
activeTools.push(tool);
|
||||
existing.add(tool.name);
|
||||
}
|
||||
}
|
||||
|
||||
function makeToolFailure(input: ToolExecutionFailure): ToolExecutionFailure {
|
||||
return input;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isTransientError(error: unknown): boolean {
|
||||
const name = error instanceof Error ? error.name : "";
|
||||
const message = getErrorMessage(error);
|
||||
return (
|
||||
name === "AbortError" ||
|
||||
name === "FetchError" ||
|
||||
/(?:timeout|timed out|ECONNRESET|ECONNREFUSED|EAI_AGAIN|ETIMEDOUT)/iu.test(
|
||||
message,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async function executeToolWithRetry<R>(
|
||||
execute: () => Promise<R>,
|
||||
options: { maxRetries: number },
|
||||
): Promise<{ result: R; retried: boolean }> {
|
||||
let attempt = 0;
|
||||
let retried = false;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
return {
|
||||
result: await execute(),
|
||||
retried,
|
||||
};
|
||||
} catch (error) {
|
||||
if (!isTransientError(error) || attempt >= options.maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
attempt += 1;
|
||||
retried = true;
|
||||
await delay(200);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function classifyToolError(
|
||||
toolName: string,
|
||||
error: unknown,
|
||||
): ToolExecutionFailure {
|
||||
const message = getErrorMessage(error);
|
||||
const isArgsInvalid =
|
||||
error instanceof Error &&
|
||||
(error.name === "ZodError" || error.constructor.name === "ZodError");
|
||||
|
||||
if (isArgsInvalid) {
|
||||
return makeToolFailure({
|
||||
kind: "args-invalid",
|
||||
message,
|
||||
retryable: true,
|
||||
toolName,
|
||||
details: error,
|
||||
});
|
||||
}
|
||||
|
||||
if (isTransientError(error)) {
|
||||
return makeToolFailure({
|
||||
kind: "transient",
|
||||
message,
|
||||
retryable: true,
|
||||
toolName,
|
||||
details: error,
|
||||
});
|
||||
}
|
||||
|
||||
return makeToolFailure({
|
||||
kind: "permanent",
|
||||
message,
|
||||
retryable: false,
|
||||
toolName,
|
||||
details: error,
|
||||
});
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function isToolErrorKind(value: unknown): value is ToolErrorKind {
|
||||
return (
|
||||
value === "args-invalid" ||
|
||||
value === "transient" ||
|
||||
value === "precondition-failed" ||
|
||||
value === "permanent"
|
||||
);
|
||||
}
|
||||
|
||||
function getPreconditionFailure(
|
||||
toolName: string,
|
||||
result: unknown,
|
||||
): ToolExecutionFailure | null {
|
||||
const record = asRecord(result);
|
||||
if (record["ok"] !== false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const kind = isToolErrorKind(record["errorKind"])
|
||||
? record["errorKind"]
|
||||
: isToolErrorKind(record["kind"])
|
||||
? record["kind"]
|
||||
: "precondition-failed";
|
||||
const message =
|
||||
typeof record["message"] === "string" && record["message"].trim()
|
||||
? record["message"]
|
||||
: "Tool precondition failed";
|
||||
|
||||
return makeToolFailure({
|
||||
kind,
|
||||
message,
|
||||
retryable: kind === "transient" || kind === "args-invalid",
|
||||
toolName,
|
||||
details: result,
|
||||
});
|
||||
}
|
||||
|
||||
function pushToolFailureMessage(
|
||||
messages: ChatMessage[],
|
||||
call: { id?: string | undefined; name: string },
|
||||
failure: ToolExecutionFailure,
|
||||
result?: unknown,
|
||||
): void {
|
||||
messages.push({
|
||||
role: "tool",
|
||||
name: call.name,
|
||||
tool_call_id: call.id,
|
||||
content: JSON.stringify(
|
||||
result ?? {
|
||||
ok: false,
|
||||
errorKind: failure.kind,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
},
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function summarizeToolResult(result: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(result).slice(0, 200);
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { Tool } from "@tools/tool";
|
||||
import { runToolPlan } from "./tool-plan-runner.js";
|
||||
import type { ToolPlan } from "./planner-schema.js";
|
||||
|
||||
function request(): AgentRequest {
|
||||
return {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "확인해줘",
|
||||
};
|
||||
}
|
||||
|
||||
describe("runToolPlan", () => {
|
||||
it("runs a small DAG and resolves arguments from request context", async () => {
|
||||
const calls: string[] = [];
|
||||
const validateTool: Tool<{ nickname: string }, { valid: boolean }> = {
|
||||
name: "validate-nickname",
|
||||
description: "닉네임 검사",
|
||||
argsSchema: z.object({
|
||||
nickname: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
valid: z.boolean(),
|
||||
}),
|
||||
execute: (args) => {
|
||||
calls.push(`validate:${args.nickname}`);
|
||||
return Promise.resolve({ valid: true });
|
||||
},
|
||||
};
|
||||
const lookupTool: Tool<{ topic: string }, { text: string }> = {
|
||||
name: "lookup-community-rule",
|
||||
description: "규칙 조회",
|
||||
argsSchema: z.object({
|
||||
topic: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
text: z.string(),
|
||||
}),
|
||||
execute: (args) => {
|
||||
calls.push(`lookup:${args.topic}`);
|
||||
return Promise.resolve({ text: "하트 규칙" });
|
||||
},
|
||||
};
|
||||
const plan: ToolPlan = {
|
||||
composeWith: "all-then-llm",
|
||||
steps: [
|
||||
{
|
||||
id: "validate",
|
||||
tool: "validate-nickname",
|
||||
argsFromContext: {
|
||||
nickname: "$request.nickname",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "rule",
|
||||
tool: "lookup-community-rule",
|
||||
argsLiteral: {
|
||||
topic: "heart",
|
||||
},
|
||||
dependsOn: ["validate"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await runToolPlan(
|
||||
plan,
|
||||
[validateTool, lookupTool],
|
||||
{},
|
||||
request(),
|
||||
);
|
||||
|
||||
expect(result.failures).toEqual([]);
|
||||
expect(result.results).toMatchObject({
|
||||
validate: { valid: true },
|
||||
rule: { text: "하트 규칙" },
|
||||
});
|
||||
expect(calls).toEqual(["validate:프리리/95/여/D", "lookup:heart"]);
|
||||
});
|
||||
|
||||
it("reports missing dependencies without running tools", async () => {
|
||||
const plan: ToolPlan = {
|
||||
composeWith: "all-then-llm",
|
||||
steps: [
|
||||
{
|
||||
id: "rule",
|
||||
tool: "lookup-community-rule",
|
||||
dependsOn: ["missing"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await runToolPlan(plan, [], {}, request());
|
||||
|
||||
expect(result.failures[0]).toMatchObject({
|
||||
kind: "permanent",
|
||||
toolName: "tool-plan",
|
||||
});
|
||||
expect(result.results).toEqual({});
|
||||
});
|
||||
|
||||
it("marks fallbackToLlm when a failing step requests fallback", async () => {
|
||||
const tool: Tool<{ topic: string }, { text: string }> = {
|
||||
name: "lookup-community-rule",
|
||||
description: "규칙 조회",
|
||||
argsSchema: z.object({
|
||||
topic: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
text: z.string(),
|
||||
}),
|
||||
execute: () => Promise.reject(new Error("rule service down")),
|
||||
};
|
||||
const plan: ToolPlan = {
|
||||
composeWith: "all-then-llm",
|
||||
steps: [
|
||||
{
|
||||
id: "rule",
|
||||
tool: "lookup-community-rule",
|
||||
argsLiteral: {
|
||||
topic: "heart",
|
||||
},
|
||||
on: {
|
||||
error: "fallback-llm",
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await runToolPlan(plan, [tool], {}, request());
|
||||
|
||||
expect(result.fallbackToLlm).toBe(true);
|
||||
expect(result.failures[0]).toMatchObject({
|
||||
kind: "permanent",
|
||||
toolName: "lookup-community-rule",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,417 @@
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import { withToolSpan } from "@obs/spans";
|
||||
import type {
|
||||
Tool,
|
||||
ToolContext,
|
||||
ToolErrorKind,
|
||||
ToolExecutionFailure,
|
||||
} from "@tools/tool";
|
||||
import type { ToolPlan, ToolPlanStep } from "./planner-schema.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface ToolPlanResult {
|
||||
durationMs: number;
|
||||
failures: ToolExecutionFailure[];
|
||||
fallbackToLlm: boolean;
|
||||
results: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface OrderedPlan {
|
||||
failures: ToolExecutionFailure[];
|
||||
levels: ToolPlanStep[][];
|
||||
}
|
||||
|
||||
function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function isTransientError(error: unknown): boolean {
|
||||
const name = error instanceof Error ? error.name : "";
|
||||
const message = getErrorMessage(error);
|
||||
return (
|
||||
name === "AbortError" ||
|
||||
name === "FetchError" ||
|
||||
/(?:timeout|timed out|ECONNRESET|ECONNREFUSED|EAI_AGAIN|ETIMEDOUT)/iu.test(
|
||||
message,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function classifyError(toolName: string, error: unknown): ToolExecutionFailure {
|
||||
const message = getErrorMessage(error);
|
||||
const isArgsInvalid =
|
||||
error instanceof Error &&
|
||||
(error.name === "ZodError" || error.constructor.name === "ZodError");
|
||||
|
||||
if (isArgsInvalid) {
|
||||
return {
|
||||
kind: "args-invalid",
|
||||
message,
|
||||
retryable: true,
|
||||
toolName,
|
||||
details: error,
|
||||
};
|
||||
}
|
||||
|
||||
if (isTransientError(error)) {
|
||||
return {
|
||||
kind: "transient",
|
||||
message,
|
||||
retryable: true,
|
||||
toolName,
|
||||
details: error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "permanent",
|
||||
message,
|
||||
retryable: false,
|
||||
toolName,
|
||||
details: error,
|
||||
};
|
||||
}
|
||||
|
||||
function makeFailure(input: {
|
||||
details?: unknown;
|
||||
kind: ToolErrorKind;
|
||||
message: string;
|
||||
retryable?: boolean;
|
||||
toolName: string;
|
||||
}): ToolExecutionFailure {
|
||||
return {
|
||||
kind: input.kind,
|
||||
message: input.message,
|
||||
retryable: input.retryable ?? false,
|
||||
toolName: input.toolName,
|
||||
...(input.details === undefined ? {} : { details: input.details }),
|
||||
};
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function isToolErrorKind(value: unknown): value is ToolErrorKind {
|
||||
return (
|
||||
value === "args-invalid" ||
|
||||
value === "transient" ||
|
||||
value === "precondition-failed" ||
|
||||
value === "permanent"
|
||||
);
|
||||
}
|
||||
|
||||
function getStructuredFailure(
|
||||
step: ToolPlanStep,
|
||||
result: unknown,
|
||||
): ToolExecutionFailure | null {
|
||||
const record = asRecord(result);
|
||||
if (record["ok"] !== false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const kind = isToolErrorKind(record["errorKind"])
|
||||
? record["errorKind"]
|
||||
: isToolErrorKind(record["kind"])
|
||||
? record["kind"]
|
||||
: "precondition-failed";
|
||||
const message =
|
||||
typeof record["message"] === "string" && record["message"].trim()
|
||||
? record["message"]
|
||||
: "Tool precondition failed";
|
||||
|
||||
return makeFailure({
|
||||
details: {
|
||||
result,
|
||||
stepId: step.id,
|
||||
},
|
||||
kind,
|
||||
message,
|
||||
retryable: kind === "args-invalid" || kind === "transient",
|
||||
toolName: step.tool,
|
||||
});
|
||||
}
|
||||
|
||||
function parsePath(path: string): string[] {
|
||||
const normalized = path.trim().replace(/^\$/u, "");
|
||||
return normalized
|
||||
.split(".")
|
||||
.flatMap((segment) => {
|
||||
const bracketIndex = segment.indexOf("[");
|
||||
if (bracketIndex === -1 || !segment.endsWith("]")) {
|
||||
return [segment];
|
||||
}
|
||||
|
||||
const key = segment.slice(0, bracketIndex);
|
||||
const index = segment.slice(bracketIndex + 1, -1);
|
||||
return key && index ? [key, index] : [segment];
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function readPath(root: unknown, path: string): unknown {
|
||||
let current = root;
|
||||
for (const segment of parsePath(path)) {
|
||||
if (Array.isArray(current)) {
|
||||
const index = Number(segment);
|
||||
if (!Number.isInteger(index)) {
|
||||
throw new Error(`Invalid array index in path: ${path}`);
|
||||
}
|
||||
current = current[index];
|
||||
continue;
|
||||
}
|
||||
|
||||
const record = asRecord(current);
|
||||
if (!(segment in record)) {
|
||||
throw new Error(`Missing context path: ${path}`);
|
||||
}
|
||||
current = record[segment];
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
function buildArgs(input: {
|
||||
ctx: ToolContext;
|
||||
request: AgentRequest;
|
||||
results: Record<string, unknown>;
|
||||
step: ToolPlanStep;
|
||||
}): Record<string, unknown> {
|
||||
const args: Record<string, unknown> = {
|
||||
...(input.step.argsLiteral ?? {}),
|
||||
};
|
||||
const roots = {
|
||||
context: input.ctx,
|
||||
request: input.request,
|
||||
results: input.results,
|
||||
session: input.ctx.session ?? null,
|
||||
tool: input.results,
|
||||
};
|
||||
|
||||
for (const [targetKey, sourcePath] of Object.entries(
|
||||
input.step.argsFromContext ?? {},
|
||||
)) {
|
||||
args[targetKey] = readPath(roots, sourcePath);
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function makePlanFailure(message: string, details?: unknown): ToolExecutionFailure {
|
||||
return makeFailure({
|
||||
details,
|
||||
kind: "permanent",
|
||||
message,
|
||||
toolName: "tool-plan",
|
||||
});
|
||||
}
|
||||
|
||||
function orderPlan(plan: ToolPlan): OrderedPlan {
|
||||
const failures: ToolExecutionFailure[] = [];
|
||||
const byId = new Map<string, ToolPlanStep>();
|
||||
for (const step of plan.steps) {
|
||||
if (byId.has(step.id)) {
|
||||
failures.push(makePlanFailure(`Duplicate toolPlan step id: ${step.id}`));
|
||||
continue;
|
||||
}
|
||||
byId.set(step.id, step);
|
||||
}
|
||||
|
||||
for (const step of plan.steps) {
|
||||
for (const dependency of step.dependsOn ?? []) {
|
||||
if (!byId.has(dependency)) {
|
||||
failures.push(
|
||||
makePlanFailure(`Missing dependency ${dependency} for step ${step.id}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
return { failures, levels: [] };
|
||||
}
|
||||
|
||||
const remaining = new Map(byId);
|
||||
const completed = new Set<string>();
|
||||
const levels: ToolPlanStep[][] = [];
|
||||
|
||||
while (remaining.size > 0) {
|
||||
const ready = [...remaining.values()].filter((step) =>
|
||||
(step.dependsOn ?? []).every((dependency) => completed.has(dependency)),
|
||||
);
|
||||
|
||||
if (ready.length === 0) {
|
||||
return {
|
||||
failures: [makePlanFailure("Cyclic toolPlan dependencies detected")],
|
||||
levels,
|
||||
};
|
||||
}
|
||||
|
||||
levels.push(ready);
|
||||
for (const step of ready) {
|
||||
remaining.delete(step.id);
|
||||
completed.add(step.id);
|
||||
}
|
||||
}
|
||||
|
||||
return { failures, levels };
|
||||
}
|
||||
|
||||
async function runStep(input: {
|
||||
ctx: ToolContext;
|
||||
request: AgentRequest;
|
||||
results: Record<string, unknown>;
|
||||
step: ToolPlanStep;
|
||||
tools: readonly Tool<unknown, unknown>[];
|
||||
}): Promise<ToolExecutionFailure | null> {
|
||||
const tool = input.tools.find((candidate) => candidate.name === input.step.tool);
|
||||
if (!tool) {
|
||||
return makeFailure({
|
||||
details: { stepId: input.step.id },
|
||||
kind: "permanent",
|
||||
message: `Tool not registered for step: ${input.step.tool}`,
|
||||
toolName: input.step.tool,
|
||||
});
|
||||
}
|
||||
|
||||
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
||||
let args: Record<string, unknown> = {};
|
||||
try {
|
||||
args = buildArgs(input);
|
||||
const validated = tool.argsSchema.parse(args);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.name": tool.name,
|
||||
"tool.plan.step_id": input.step.id,
|
||||
},
|
||||
() => tool.execute(validated, input.ctx),
|
||||
);
|
||||
const structuredFailure = getStructuredFailure(input.step, result);
|
||||
if (structuredFailure) {
|
||||
traceStep?.finish({
|
||||
args,
|
||||
errorKind: structuredFailure.kind,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: structuredFailure.message,
|
||||
});
|
||||
return structuredFailure;
|
||||
}
|
||||
|
||||
input.results[input.step.id] = result;
|
||||
traceStep?.finish({
|
||||
args,
|
||||
name: tool.name,
|
||||
ok: true,
|
||||
resultSummary: summarizeToolResult(result),
|
||||
});
|
||||
return null;
|
||||
} catch (error) {
|
||||
const failure = classifyError(tool.name, error);
|
||||
traceStep?.finish({
|
||||
args,
|
||||
errorKind: failure.kind,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: failure.message,
|
||||
});
|
||||
return failure;
|
||||
}
|
||||
}
|
||||
|
||||
function shouldFallback(step: ToolPlanStep): boolean {
|
||||
return step.on?.error === "fallback-llm";
|
||||
}
|
||||
|
||||
async function runLevel(input: {
|
||||
ctx: ToolContext;
|
||||
failures: ToolExecutionFailure[];
|
||||
request: AgentRequest;
|
||||
results: Record<string, unknown>;
|
||||
steps: ToolPlanStep[];
|
||||
tools: readonly Tool<unknown, unknown>[];
|
||||
}): Promise<{ abort: boolean; fallbackToLlm: boolean }> {
|
||||
let abort = false;
|
||||
let fallbackToLlm = false;
|
||||
|
||||
for (let index = 0; index < input.steps.length; index += 2) {
|
||||
const chunk = input.steps.slice(index, index + 2);
|
||||
const failures = await Promise.all(
|
||||
chunk.map((step) =>
|
||||
runStep({
|
||||
ctx: input.ctx,
|
||||
request: input.request,
|
||||
results: input.results,
|
||||
step,
|
||||
tools: input.tools,
|
||||
}).then((failure) => ({ failure, step })),
|
||||
),
|
||||
);
|
||||
|
||||
for (const { failure, step } of failures) {
|
||||
if (!failure) {
|
||||
continue;
|
||||
}
|
||||
input.failures.push(failure);
|
||||
if (shouldFallback(step)) {
|
||||
fallbackToLlm = true;
|
||||
}
|
||||
if (step.on?.error === "abort" || !step.on?.error) {
|
||||
abort = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (abort || fallbackToLlm) {
|
||||
return { abort, fallbackToLlm };
|
||||
}
|
||||
}
|
||||
|
||||
return { abort, fallbackToLlm };
|
||||
}
|
||||
|
||||
export async function runToolPlan(
|
||||
plan: ToolPlan,
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
ctx: ToolContext,
|
||||
request: AgentRequest,
|
||||
): Promise<ToolPlanResult> {
|
||||
const startedAt = Date.now();
|
||||
const results: Record<string, unknown> = {};
|
||||
const ordered = orderPlan(plan);
|
||||
const failures = [...ordered.failures];
|
||||
let fallbackToLlm = false;
|
||||
|
||||
for (const level of ordered.levels) {
|
||||
const levelResult = await runLevel({
|
||||
ctx,
|
||||
failures,
|
||||
request,
|
||||
results,
|
||||
steps: level,
|
||||
tools,
|
||||
});
|
||||
|
||||
fallbackToLlm = fallbackToLlm || levelResult.fallbackToLlm;
|
||||
if (levelResult.abort || levelResult.fallbackToLlm) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
durationMs: Math.max(0, Date.now() - startedAt),
|
||||
failures,
|
||||
fallbackToLlm,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeToolResult(result: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(result).slice(0, 200);
|
||||
} catch {
|
||||
return String(result).slice(0, 200);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,9 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
|
||||
import type { AbstentionDecision } from "./abstention.js";
|
||||
import type { RoutingDecision } from "./router.js";
|
||||
import type { ToolErrorKind } from "@tools/tool";
|
||||
|
||||
export type ProcessTraceStepName =
|
||||
| "classify"
|
||||
| "critic"
|
||||
@@ -19,6 +23,7 @@ export interface ProcessTraceStep<T> {
|
||||
export interface ProcessTraceSnapshot {
|
||||
steps: {
|
||||
classify?: ProcessTraceStep<{
|
||||
abstention?: AbstentionDecision | undefined;
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: "regex" | "llm";
|
||||
@@ -26,6 +31,7 @@ export interface ProcessTraceSnapshot {
|
||||
derive?: ProcessTraceStep<{
|
||||
phase: string;
|
||||
responseMode: string;
|
||||
routingDecision?: RoutingDecision | undefined;
|
||||
shouldRespond: boolean;
|
||||
tier: 0 | 1 | 2;
|
||||
}> | undefined;
|
||||
@@ -33,14 +39,18 @@ export interface ProcessTraceSnapshot {
|
||||
approach: string;
|
||||
mustAvoid: string[];
|
||||
mustInclude: string[];
|
||||
requestResources?: string[] | undefined;
|
||||
suggestedTools: string[];
|
||||
targetLengthChars: number;
|
||||
toolPlanStepCount?: number | undefined;
|
||||
}> | undefined;
|
||||
toolCalls?: Array<
|
||||
ProcessTraceStep<{
|
||||
args: unknown;
|
||||
errorKind?: ToolErrorKind | undefined;
|
||||
name: string;
|
||||
ok: boolean;
|
||||
retried?: boolean | undefined;
|
||||
resultSummary: string;
|
||||
}>
|
||||
> | undefined;
|
||||
@@ -50,8 +60,15 @@ export interface ProcessTraceSnapshot {
|
||||
tokensOut: number;
|
||||
}> | undefined;
|
||||
critic?: ProcessTraceStep<{
|
||||
abstained?: boolean | undefined;
|
||||
decision: "keep" | "rewrite";
|
||||
iterations?: number | undefined;
|
||||
rationale: string;
|
||||
rubricScores?: {
|
||||
accuracy: number;
|
||||
scope: number;
|
||||
tone: number;
|
||||
} | undefined;
|
||||
}> | undefined;
|
||||
validate?: ProcessTraceStep<{
|
||||
reasons: string[];
|
||||
|
||||
Reference in new issue
Block a user