Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
@@ -0,0 +1,83 @@
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import {
|
||||
getCriticMaxTokens,
|
||||
getLlmTemperature,
|
||||
getTier2CriticModel,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
export interface CritiqueResult {
|
||||
changed: boolean;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function serializeMessages(messages: ChatMessage[]): string {
|
||||
return messages
|
||||
.map((message) => `[${message.role}] ${message.content}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function buildCriticMessages(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): ChatMessage[] {
|
||||
return [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"당신은 온보딩 응답을 검토하는 critic입니다.",
|
||||
"초안이 충분히 정확하고 범위를 벗어나지 않으며 간결하면 반드시 [OK]만 출력하세요.",
|
||||
"고쳐야 한다면 유저에게 바로 보낼 최종 한국어 답변만 출력하고, 설명이나 해설은 쓰지 마세요.",
|
||||
"질문에서 묻지 않은 추가 절차를 길게 덧붙이지 말고, 꼭 필요한 내용만 남기세요.",
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}`,
|
||||
"[CONVERSATION]",
|
||||
serializeMessages(messages),
|
||||
"[DRAFT]",
|
||||
draft,
|
||||
].join("\n\n"),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
export class OnboardingCritic {
|
||||
constructor(private readonly llmClient: LlmClient) {}
|
||||
|
||||
async critique(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options?: { deadlineMs?: number | undefined },
|
||||
): Promise<CritiqueResult> {
|
||||
const response = await this.llmClient.chatCompletion(
|
||||
buildCriticMessages(draft, derived, messages),
|
||||
{
|
||||
deadlineMs: options?.deadlineMs,
|
||||
maxTokens: getCriticMaxTokens(),
|
||||
modelOverride: getTier2CriticModel(),
|
||||
role: "critic",
|
||||
temperature: Math.max(0, getLlmTemperature() - 0.15),
|
||||
tier: 2,
|
||||
},
|
||||
);
|
||||
|
||||
const text = response?.content.trim() ?? "";
|
||||
if (!text || text.startsWith("[OK]")) {
|
||||
return {
|
||||
changed: false,
|
||||
text: draft,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
changed: true,
|
||||
text,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { OnboardingOrchestrator } from "./orchestrator.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
|
||||
return {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "help-how" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 256,
|
||||
allowShortResponse: false,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("OnboardingOrchestrator", () => {
|
||||
it("uses direct response path for tier 0", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
return Promise.resolve({
|
||||
content: "안내 도와드릴게요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "안내 도와드릴게요." } }] },
|
||||
message: { role: "assistant" as const, content: "안내 도와드릴게요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 0 }),
|
||||
[{ role: "user", content: "도와줘" }],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("안내 도와드릴게요.");
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]?.options?.role).toBe("responder");
|
||||
});
|
||||
|
||||
it("runs planner then responder for tier 1", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "redirect-to-step",
|
||||
targetPhase: "photo-pending",
|
||||
mustInclude: ["하트 인증만 안내"],
|
||||
mustAvoid: ["디스코드 절차"],
|
||||
suggestedTools: [],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "질문 범위만 답하기",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
|
||||
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 1 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "이제 뭐 남았지" },
|
||||
],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("하트 인증만 남았어요.");
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls[0]?.options?.role).toBe("planner");
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[PLAN]");
|
||||
});
|
||||
|
||||
it("runs critic after responder for tier 2 and applies rewrite", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "answer-direct",
|
||||
targetPhase: "photo-pending",
|
||||
mustInclude: [],
|
||||
mustAvoid: ["추가 절차 나열"],
|
||||
suggestedTools: [],
|
||||
tone: "concise",
|
||||
targetLengthChars: 60,
|
||||
rationale: "짧게 답하기",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
if (calls.length === 2) {
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: {
|
||||
choices: [
|
||||
{ message: { content: "하트 인증만 남았어요. 커버 보기로 들어가서..." } },
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant" as const,
|
||||
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
|
||||
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 2 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "이제 뭐 남았지" },
|
||||
],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("하트 인증만 남았어요.");
|
||||
expect(calls).toHaveLength(3);
|
||||
expect(calls[2]?.options?.role).toBe("critic");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,382 @@
|
||||
import { getContext } from "@obs/context";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import {
|
||||
getLlmTemperature,
|
||||
getPlannerMaxTokens,
|
||||
getTier0EstimatedCallMs,
|
||||
getTier0MaxTokens,
|
||||
getTier0Model,
|
||||
getTier1EstimatedCallMs,
|
||||
getTier1MaxTokens,
|
||||
getTier1PlannerModel,
|
||||
getTier1ResponderModel,
|
||||
getTier2EstimatedCallMs,
|
||||
getTier2MaxTokens,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
import type { PlannerOutput } from "./planner-schema.js";
|
||||
import { PlannerOutputSchema } from "./planner-schema.js";
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
|
||||
export interface OrchestratorRunOptions {
|
||||
maxTokens?: number | undefined;
|
||||
temperature?: number | undefined;
|
||||
toolContext: ToolContext;
|
||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||
}
|
||||
|
||||
function cloneMessages(messages: ChatMessage[]): ChatMessage[] {
|
||||
return messages.map((message) => ({
|
||||
...message,
|
||||
tool_calls: message.tool_calls ? [...message.tool_calls] : undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
function injectSystemAddendum(
|
||||
messages: ChatMessage[],
|
||||
addendum: string,
|
||||
): ChatMessage[] {
|
||||
const cloned = cloneMessages(messages);
|
||||
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
|
||||
|
||||
if (firstSystemIndex === -1) {
|
||||
cloned.unshift({
|
||||
role: "system",
|
||||
content: addendum,
|
||||
});
|
||||
return cloned;
|
||||
}
|
||||
|
||||
const current = cloned[firstSystemIndex];
|
||||
if (!current) {
|
||||
return cloned;
|
||||
}
|
||||
|
||||
cloned[firstSystemIndex] = {
|
||||
...current,
|
||||
content: `${current.content}\n\n${addendum}`,
|
||||
};
|
||||
return cloned;
|
||||
}
|
||||
|
||||
function summarizeMessages(messages: ChatMessage[]): string {
|
||||
return messages
|
||||
.map((message) => `[${message.role}] ${message.content}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function tryParsePlannerOutput(raw: string): PlannerOutput | null {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return PlannerOutputSchema.parse(parsed);
|
||||
} catch {
|
||||
const start = raw.indexOf("{");
|
||||
const end = raw.lastIndexOf("}");
|
||||
if (start === -1 || end === -1 || end <= start) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw.slice(start, end + 1)) as unknown;
|
||||
return PlannerOutputSchema.parse(parsed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getDefaultPlan(derived: DerivedState): PlannerOutput {
|
||||
return {
|
||||
approach:
|
||||
derived.responseMode === "llm-with-tools"
|
||||
? "validate-then-explain"
|
||||
: derived.intent.kind === "help-how" || derived.intent.kind === "help-where"
|
||||
? "redirect-to-step"
|
||||
: "answer-direct",
|
||||
targetPhase: derived.phase.kind,
|
||||
mustInclude: [],
|
||||
mustAvoid: [],
|
||||
suggestedTools:
|
||||
derived.responseMode === "llm-with-tools" ? ["validate-nickname"] : [],
|
||||
tone: derived.allowShortResponse ? "concise" : "friendly",
|
||||
targetLengthChars: derived.allowShortResponse ? 80 : 150,
|
||||
rationale: "fallback-plan",
|
||||
};
|
||||
}
|
||||
|
||||
function getRemainingDeadlineMs(): number | undefined {
|
||||
const deadlineAt = getContext()?.deadlineAt;
|
||||
if (!deadlineAt) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const deadlineMs = Date.parse(deadlineAt) - Date.now();
|
||||
return Number.isFinite(deadlineMs) ? deadlineMs : undefined;
|
||||
}
|
||||
|
||||
function canAffordCall(estimatedCallMs: number): boolean {
|
||||
const remainingMs = getRemainingDeadlineMs();
|
||||
return remainingMs === undefined || remainingMs > estimatedCallMs;
|
||||
}
|
||||
|
||||
function buildPlannerMessages(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): ChatMessage[] {
|
||||
return [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"당신은 온보딩 응답을 설계하는 planner입니다.",
|
||||
"반드시 JSON만 출력하세요.",
|
||||
"질문에 직접 답할 수 있으면 answer-direct를 우선하고, 유저가 묻지 않은 추가 절차는 mustAvoid에 넣으세요.",
|
||||
"tool은 꼭 필요한 경우에만 suggestedTools에 넣고 최대 3개까지만 고르세요.",
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}, tier=${derived.tier}`,
|
||||
"[CONVERSATION]",
|
||||
summarizeMessages(messages),
|
||||
"[SCHEMA]",
|
||||
JSON.stringify({
|
||||
approach:
|
||||
"answer-direct | ask-clarification | redirect-to-step | validate-then-explain | lookup-rule-then-answer | soft-decline",
|
||||
targetPhase:
|
||||
"fresh | nickname-pending | photo-pending | both-pending | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
|
||||
mustInclude: ["short strings"],
|
||||
mustAvoid: ["short strings"],
|
||||
suggestedTools: [
|
||||
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context",
|
||||
],
|
||||
tone: "friendly | concise | encouraging | neutral | apologetic",
|
||||
targetLengthChars: 150,
|
||||
rationale: "short reason",
|
||||
}),
|
||||
].join("\n\n"),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function buildResponderAddendum(plan: PlannerOutput): string {
|
||||
const lines = [
|
||||
"[PLAN]",
|
||||
`approach=${plan.approach}`,
|
||||
`tone=${plan.tone}`,
|
||||
`targetLengthChars=${plan.targetLengthChars}`,
|
||||
];
|
||||
|
||||
if (plan.targetPhase) {
|
||||
lines.push(`targetPhase=${plan.targetPhase}`);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
`mustInclude=${plan.mustInclude.length > 0 ? plan.mustInclude.join(" | ") : "-"}`,
|
||||
`mustAvoid=${plan.mustAvoid.length > 0 ? plan.mustAvoid.join(" | ") : "-"}`,
|
||||
"위 계획을 참고하되, 실제 유저에게 보낼 최종 한국어 답변만 출력하세요.",
|
||||
);
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function pickToolsForPlan(
|
||||
plan: PlannerOutput,
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
): readonly Tool<unknown, unknown>[] {
|
||||
if (plan.approach === "answer-direct" || plan.approach === "soft-decline") {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (plan.suggestedTools.length === 0) {
|
||||
return tools;
|
||||
}
|
||||
|
||||
const picked = tools.filter((tool) => plan.suggestedTools.includes(tool.name as never));
|
||||
return picked.length > 0 ? picked : tools;
|
||||
}
|
||||
|
||||
export class OnboardingOrchestrator {
|
||||
private readonly critic: OnboardingCritic;
|
||||
|
||||
constructor(private readonly llmClient: LlmClient) {
|
||||
this.critic = new OnboardingCritic(llmClient);
|
||||
}
|
||||
|
||||
private async runDirect(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
responderTools: readonly Tool<unknown, unknown>[],
|
||||
maxTokens: number,
|
||||
modelOverride: string,
|
||||
): Promise<string | null> {
|
||||
const deadlineMs = getRemainingDeadlineMs();
|
||||
|
||||
if (responderTools.length > 0) {
|
||||
return runWithTools({
|
||||
deadlineMs,
|
||||
llmClient: this.llmClient,
|
||||
maxTokens,
|
||||
messages,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
tier: derived.tier,
|
||||
temperature: options.temperature ?? getLlmTemperature(),
|
||||
toolContext: options.toolContext,
|
||||
tools: responderTools,
|
||||
});
|
||||
}
|
||||
|
||||
const response = await this.llmClient.chatCompletion(messages, {
|
||||
deadlineMs,
|
||||
maxTokens,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
temperature: options.temperature ?? getLlmTemperature(),
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
return response?.content ?? null;
|
||||
}
|
||||
|
||||
private async runPlanner(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): Promise<PlannerOutput> {
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
jsonMode: true,
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
temperature: 0,
|
||||
tier: 1,
|
||||
});
|
||||
|
||||
const firstPlan = firstAttempt?.content
|
||||
? tryParsePlannerOutput(firstAttempt.content)
|
||||
: null;
|
||||
if (firstPlan) {
|
||||
return firstPlan;
|
||||
}
|
||||
|
||||
const retryMessages = [
|
||||
...plannerMessages,
|
||||
{
|
||||
role: "user" as const,
|
||||
content:
|
||||
"방금 응답은 스키마를 벗어났어요. 설명 없이 JSON object 하나만 다시 출력하세요.",
|
||||
},
|
||||
];
|
||||
|
||||
const retry = await this.llmClient.chatCompletion(retryMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
jsonMode: true,
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
temperature: 0,
|
||||
tier: 1,
|
||||
});
|
||||
|
||||
const retryPlan = retry?.content ? tryParsePlannerOutput(retry.content) : null;
|
||||
return retryPlan ?? getDefaultPlan(derived);
|
||||
}
|
||||
|
||||
private async runTier0(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
const maxTokens = options.maxTokens ?? getTier0MaxTokens();
|
||||
const tools =
|
||||
derived.responseMode === "llm-with-tools" ? (options.tools ?? []) : [];
|
||||
|
||||
return this.runDirect(
|
||||
derived,
|
||||
messages,
|
||||
options,
|
||||
tools,
|
||||
maxTokens,
|
||||
getTier0Model(),
|
||||
);
|
||||
}
|
||||
|
||||
private async runTier1(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
const plan = await this.runPlanner(derived, messages);
|
||||
const responderMessages = injectSystemAddendum(
|
||||
messages,
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
|
||||
|
||||
return this.runDirect(
|
||||
derived,
|
||||
responderMessages,
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier1MaxTokens(),
|
||||
getTier1ResponderModel(),
|
||||
);
|
||||
}
|
||||
|
||||
private async runTier2(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
const plan = await this.runPlanner(derived, messages);
|
||||
const responderMessages = injectSystemAddendum(
|
||||
messages,
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
|
||||
|
||||
const draft = await this.runDirect(
|
||||
derived,
|
||||
responderMessages,
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier2MaxTokens(),
|
||||
getTier1ResponderModel(),
|
||||
);
|
||||
|
||||
if (!draft) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const critique = await this.critic.critique(draft, derived, responderMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
});
|
||||
return critique.text;
|
||||
}
|
||||
|
||||
async run(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
if (derived.tier >= 2 && canAffordCall(getTier2EstimatedCallMs())) {
|
||||
return this.runTier2(derived, messages, options);
|
||||
}
|
||||
|
||||
if (derived.tier >= 1 && canAffordCall(getTier1EstimatedCallMs())) {
|
||||
return this.runTier1(derived, messages, options);
|
||||
}
|
||||
|
||||
if (!canAffordCall(getTier0EstimatedCallMs())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.runTier0(derived, messages, options);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const PlannerApproachSchema = z.enum([
|
||||
"answer-direct",
|
||||
"ask-clarification",
|
||||
"redirect-to-step",
|
||||
"validate-then-explain",
|
||||
"lookup-rule-then-answer",
|
||||
"soft-decline",
|
||||
]);
|
||||
|
||||
export const PlannerTargetPhaseSchema = z.enum([
|
||||
"fresh",
|
||||
"nickname-pending",
|
||||
"photo-pending",
|
||||
"both-pending",
|
||||
"nickname-rejected",
|
||||
"photo-rejected",
|
||||
"deadline-near",
|
||||
"completed",
|
||||
"timed-out",
|
||||
]);
|
||||
|
||||
export const PlannerSuggestedToolSchema = z.enum([
|
||||
"validate-nickname",
|
||||
"check-onboarding-progress",
|
||||
"lookup-community-rule",
|
||||
"get-room-context",
|
||||
]);
|
||||
|
||||
export const PlannerToneSchema = z.enum([
|
||||
"friendly",
|
||||
"concise",
|
||||
"encouraging",
|
||||
"neutral",
|
||||
"apologetic",
|
||||
]);
|
||||
|
||||
export const PlannerOutputSchema = z.object({
|
||||
approach: PlannerApproachSchema,
|
||||
targetPhase: PlannerTargetPhaseSchema.optional(),
|
||||
mustInclude: z.array(z.string().min(1).max(80)).max(5).default([]),
|
||||
mustAvoid: z.array(z.string().min(1).max(80)).max(5).default([]),
|
||||
suggestedTools: z.array(PlannerSuggestedToolSchema).max(3).default([]),
|
||||
tone: PlannerToneSchema.default("friendly"),
|
||||
targetLengthChars: z.number().int().min(20).max(500).default(150),
|
||||
rationale: z.string().max(200),
|
||||
});
|
||||
|
||||
export type PlannerOutput = z.infer<typeof PlannerOutputSchema>;
|
||||
@@ -0,0 +1,219 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
import type { Tool } from "@tools/tool";
|
||||
|
||||
describe("runWithTools", () => {
|
||||
it("executes a tool call and returns the model's final answer", async () => {
|
||||
const calls: Array<{
|
||||
content: string;
|
||||
messages: Array<{ role: string; content: string }>;
|
||||
options?: Record<string, unknown> | undefined;
|
||||
}> = [];
|
||||
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
options?: Record<string, unknown>,
|
||||
) => {
|
||||
calls.push({
|
||||
content: messages[messages.length - 1]?.content ?? "",
|
||||
messages,
|
||||
options,
|
||||
});
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = {
|
||||
name: "validate-nickname",
|
||||
description: "닉네임 형식 검사",
|
||||
argsSchema: z.object({
|
||||
nickname: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
valid: z.boolean(),
|
||||
issues: z.array(z.string()),
|
||||
}),
|
||||
execute: () =>
|
||||
Promise.resolve({
|
||||
valid: false,
|
||||
issues: ["티어는 '/'로 구분해서 넣어주세요."],
|
||||
}),
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "도구를 필요할 때만 사용해.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "닉네임이 왜 안 돼요?",
|
||||
},
|
||||
],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("프리리/95/여/D");
|
||||
expect(calls[0]?.options?.tools).toHaveLength(1);
|
||||
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves native tool call responses for follow-up turns", async () => {
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
seenMessages.push(messages);
|
||||
|
||||
if (seenMessages.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: "",
|
||||
endpoint: "mock://cliproxy",
|
||||
provider: "cliproxy",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: {
|
||||
name: "lookup-community-rule",
|
||||
arguments: '{"topic":"age"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "",
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: {
|
||||
name: "lookup-community-rule",
|
||||
arguments: '{"topic":"age"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "나이 기준은 27세 이상이에요.",
|
||||
endpoint: "mock://cliproxy",
|
||||
provider: "cliproxy",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: "나이 기준은 27세 이상이에요.",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "나이 기준은 27세 이상이에요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ topic: string }, { topic: string; text: string }> = {
|
||||
name: "lookup-community-rule",
|
||||
description: "커뮤니티 규칙 조회",
|
||||
argsSchema: z.object({
|
||||
topic: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
topic: z.string(),
|
||||
text: z.string(),
|
||||
}),
|
||||
execute: () =>
|
||||
Promise.resolve({
|
||||
topic: "age",
|
||||
text: "나이 기준은 27세 이상이에요.",
|
||||
}),
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "필요할 때만 도구를 사용해.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "나이 기준이 뭐예요?",
|
||||
},
|
||||
],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toBe("나이 기준은 27세 이상이에요.");
|
||||
expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true);
|
||||
expect(
|
||||
seenMessages[1]?.some(
|
||||
(message) => Array.isArray(message.tool_calls) && message.role === "assistant",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,89 @@
|
||||
import { withToolSpan } from "@obs/spans";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { parseToolCalls } from "@tools/parser";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
|
||||
export interface ToolLoopOptions {
|
||||
deadlineMs?: number | undefined;
|
||||
llmClient: LlmClient;
|
||||
maxHops?: number | undefined;
|
||||
maxTokens?: number | undefined;
|
||||
messages: ChatMessage[];
|
||||
modelOverride?: string | undefined;
|
||||
role?: ChatCompletionOptions["role"];
|
||||
signal?: AbortSignal | undefined;
|
||||
temperature?: number | undefined;
|
||||
tier?: ChatCompletionOptions["tier"];
|
||||
toolContext: ToolContext;
|
||||
tools: readonly Tool<unknown, unknown>[];
|
||||
}
|
||||
|
||||
export async function runWithTools(
|
||||
options: ToolLoopOptions,
|
||||
): Promise<string | null> {
|
||||
const maxHops = options.maxHops ?? 2;
|
||||
const messages = options.messages.map((message) => ({ ...message }));
|
||||
let allowNativeTools = true;
|
||||
|
||||
for (let hop = 0; hop <= maxHops; hop++) {
|
||||
const response = await options.llmClient.chatCompletion(messages, {
|
||||
deadlineMs: options.deadlineMs,
|
||||
maxTokens: options.maxTokens,
|
||||
modelOverride: options.modelOverride,
|
||||
role: options.role,
|
||||
signal: options.signal,
|
||||
temperature: options.temperature,
|
||||
tier: options.tier,
|
||||
tools: allowNativeTools ? options.tools : undefined,
|
||||
});
|
||||
|
||||
if (!response) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const toolCalls = parseToolCalls(response.message);
|
||||
if (toolCalls.length === 0) {
|
||||
return response.content;
|
||||
}
|
||||
|
||||
if (hop === maxHops) {
|
||||
messages.push(response.message);
|
||||
messages.push({
|
||||
role: "user",
|
||||
content:
|
||||
"도구 호출은 여기까지 하고, 지금까지 얻은 정보만으로 한국어 문장으로 최종 답변해줘.",
|
||||
});
|
||||
allowNativeTools = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
messages.push(response.message);
|
||||
|
||||
for (const call of toolCalls) {
|
||||
const tool = options.tools.find((candidate) => candidate.name === call.name);
|
||||
if (!tool) {
|
||||
throw new Error(`Tool not registered for call: ${call.name}`);
|
||||
}
|
||||
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
|
||||
messages.push({
|
||||
role: "tool",
|
||||
name: tool.name,
|
||||
tool_call_id: call.id,
|
||||
content: JSON.stringify(result),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { validateOnboardingResponse } from "./validator.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
function makeDerived(
|
||||
overrides?: Partial<DerivedState>,
|
||||
): DerivedState {
|
||||
return {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "photo-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 256,
|
||||
allowShortResponse: false,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("validateOnboardingResponse", () => {
|
||||
it("rejects discord guidance before onboarding is completed", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"디스코드 서버에 들어가시면 돼요.",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.status).toBe("reject");
|
||||
expect(result.reasons).toContain("stateConsistency");
|
||||
});
|
||||
|
||||
it("rejects invalid nickname examples", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"홍길동/95/남/Z로 바꾸시면 돼요.",
|
||||
makeDerived({
|
||||
phase: { kind: "nickname-rejected", current: "홍길동/95/남/골드", issues: [] },
|
||||
intent: { kind: "nickname-question" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.status).toBe("reject");
|
||||
expect(result.reasons).toContain("nicknameExampleConsistency");
|
||||
});
|
||||
|
||||
it("soft-warns on repeated phrases", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"하트 인증 해주세요 하트 인증 해주세요 하트 인증 해주세요",
|
||||
makeDerived(),
|
||||
{ allowShortResponse: true },
|
||||
);
|
||||
|
||||
expect(result.status).toBe("soft-warn");
|
||||
expect(result.reasons).toContain("phraseRepetition");
|
||||
});
|
||||
|
||||
it("passes short status responses when allowShortResponse is enabled", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
makeDerived({
|
||||
intent: { kind: "status-question" },
|
||||
}),
|
||||
{ allowShortResponse: true },
|
||||
);
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { getMaxResponseLength } from "../../../config.js";
|
||||
import { VALID_TIER_SET } from "../constants.js";
|
||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
const ALLOWED_URL_PATTERNS = [
|
||||
/discord\.gg\/lolympus/iu,
|
||||
/open\.kakao\.com\/o\/gOscBAEh/iu,
|
||||
];
|
||||
|
||||
export interface ValidationResult {
|
||||
status: "pass" | "soft-warn" | "reject";
|
||||
text: string;
|
||||
reasons: string[];
|
||||
}
|
||||
|
||||
function postProcessResponse(text: string): string {
|
||||
let processed = text.trim();
|
||||
|
||||
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
|
||||
|
||||
processed = processed.replace(
|
||||
/https?:\/\/[^\s)}\]]+/gu,
|
||||
(match) => {
|
||||
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
|
||||
return match;
|
||||
}
|
||||
return "";
|
||||
},
|
||||
);
|
||||
|
||||
processed = processed.replace(/ +/gu, " ").trim();
|
||||
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1");
|
||||
processed = processed.replace(/\*(.+?)\*/gu, "$1");
|
||||
processed = processed.replace(/__(.+?)__/gu, "$1");
|
||||
processed = processed.replace(/~~(.+?)~~/gu, "$1");
|
||||
processed = processed.replace(/^#{1,6}\s+/gmu, "");
|
||||
processed = processed.replace(/^[-*]\s+/gmu, "");
|
||||
processed = processed.replace(/`(.+?)`/gu, "$1");
|
||||
|
||||
const maxLen = getMaxResponseLength();
|
||||
if (processed.length > maxLen) {
|
||||
processed = processed.slice(0, maxLen).trimEnd();
|
||||
const sentenceEndPattern =
|
||||
/(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
|
||||
|
||||
for (let index = processed.length; index > maxLen * 0.5; index--) {
|
||||
const sub = processed.slice(0, index);
|
||||
if (sentenceEndPattern.test(sub)) {
|
||||
processed = sub;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return processed.trim();
|
||||
}
|
||||
|
||||
function findInvalidNicknameExamples(text: string): string[] {
|
||||
const matches = [
|
||||
...text.matchAll(/([^\s/]{1,20})\/(\d{2})\/(남|여)\/([A-Z]{1,2})/gu),
|
||||
];
|
||||
|
||||
return matches.flatMap((match) => {
|
||||
const tier = match[4];
|
||||
if (!tier || VALID_TIER_SET.has(tier)) {
|
||||
return [];
|
||||
}
|
||||
return [tier];
|
||||
});
|
||||
}
|
||||
|
||||
function hasIntentKeywordMatch(text: string, derived: DerivedState): boolean {
|
||||
const keywords = EXPECTED_KEYWORDS[derived.intent.kind];
|
||||
if (keywords.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const lower = text.toLowerCase();
|
||||
return keywords.some((keyword) => lower.includes(keyword.toLowerCase()));
|
||||
}
|
||||
|
||||
function hasRepeatedPhrase(text: string): boolean {
|
||||
const words = text.split(/\s+/u);
|
||||
if (words.length < 6) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const phrases = new Map<string, number>();
|
||||
for (let index = 0; index < words.length - 1; index++) {
|
||||
const bigram = `${words[index]} ${words[index + 1]}`;
|
||||
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
|
||||
}
|
||||
|
||||
return [...phrases.values()].some((count) => count >= 3);
|
||||
}
|
||||
|
||||
function markValidation(result: ValidationResult): ValidationResult {
|
||||
const activeSpan = trace.getActiveSpan();
|
||||
activeSpan?.setAttributes({
|
||||
"agent.validator.status": result.status,
|
||||
"agent.validator.warnings": result.reasons,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export function validateOnboardingResponse(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
options?: { allowShortResponse?: boolean | undefined },
|
||||
): ValidationResult {
|
||||
const text = postProcessResponse(draft);
|
||||
const rejectReasons: string[] = [];
|
||||
const softWarnReasons: string[] = [];
|
||||
|
||||
if (!text) {
|
||||
rejectReasons.push("empty");
|
||||
}
|
||||
|
||||
if (text.includes("[NO_RESPONSE]")) {
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text: "[NO_RESPONSE]",
|
||||
reasons: [],
|
||||
});
|
||||
}
|
||||
|
||||
if (!options?.allowShortResponse && text.length < 10) {
|
||||
rejectReasons.push("minLength");
|
||||
}
|
||||
|
||||
const mentionsDiscord = /(디스코드|discord|디코)/iu.test(text);
|
||||
const advancesDiscordStep = /(서버|프로필|권한|입장)/iu.test(text);
|
||||
const defersDiscordStep =
|
||||
/(먼저|완료.*후|완료하신 뒤|완료하신 후|모두 완료|완료되면)/u.test(text);
|
||||
|
||||
if (
|
||||
mentionsDiscord &&
|
||||
derived.phase.kind !== "completed" &&
|
||||
advancesDiscordStep &&
|
||||
!defersDiscordStep
|
||||
) {
|
||||
rejectReasons.push("stateConsistency");
|
||||
}
|
||||
|
||||
if (findInvalidNicknameExamples(text).length > 0) {
|
||||
rejectReasons.push("nicknameExampleConsistency");
|
||||
}
|
||||
|
||||
if (!hasIntentKeywordMatch(text, derived)) {
|
||||
softWarnReasons.push("intentAlignment");
|
||||
}
|
||||
|
||||
if (hasRepeatedPhrase(text)) {
|
||||
softWarnReasons.push("phraseRepetition");
|
||||
}
|
||||
|
||||
if (rejectReasons.length > 0) {
|
||||
return markValidation({
|
||||
status: "reject",
|
||||
text,
|
||||
reasons: [...rejectReasons, ...softWarnReasons],
|
||||
});
|
||||
}
|
||||
|
||||
if (softWarnReasons.length > 0) {
|
||||
return markValidation({
|
||||
status: "soft-warn",
|
||||
text,
|
||||
reasons: softWarnReasons,
|
||||
});
|
||||
}
|
||||
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text,
|
||||
reasons: [],
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user