Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+83
View File
@@ -0,0 +1,83 @@
import type { ChatMessage } from "@llm/types";
import type { LlmClient } from "@llm/llm-client";
import {
getCriticMaxTokens,
getLlmTemperature,
getTier2CriticModel,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
export interface CritiqueResult {
changed: boolean;
text: string;
}
function serializeMessages(messages: ChatMessage[]): string {
return messages
.map((message) => `[${message.role}] ${message.content}`)
.join("\n\n");
}
function buildCriticMessages(
draft: string,
derived: DerivedState,
messages: ChatMessage[],
): ChatMessage[] {
return [
{
role: "system",
content: [
"당신은 온보딩 응답을 검토하는 critic입니다.",
"초안이 충분히 정확하고 범위를 벗어나지 않으며 간결하면 반드시 [OK]만 출력하세요.",
"고쳐야 한다면 유저에게 바로 보낼 최종 한국어 답변만 출력하고, 설명이나 해설은 쓰지 마세요.",
"질문에서 묻지 않은 추가 절차를 길게 덧붙이지 말고, 꼭 필요한 내용만 남기세요.",
].join("\n"),
},
{
role: "user",
content: [
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}`,
"[CONVERSATION]",
serializeMessages(messages),
"[DRAFT]",
draft,
].join("\n\n"),
},
];
}
export class OnboardingCritic {
constructor(private readonly llmClient: LlmClient) {}
async critique(
draft: string,
derived: DerivedState,
messages: ChatMessage[],
options?: { deadlineMs?: number | undefined },
): Promise<CritiqueResult> {
const response = await this.llmClient.chatCompletion(
buildCriticMessages(draft, derived, messages),
{
deadlineMs: options?.deadlineMs,
maxTokens: getCriticMaxTokens(),
modelOverride: getTier2CriticModel(),
role: "critic",
temperature: Math.max(0, getLlmTemperature() - 0.15),
tier: 2,
},
);
const text = response?.content.trim() ?? "";
if (!text || text.startsWith("[OK]")) {
return {
changed: false,
text: draft,
};
}
return {
changed: true,
text,
};
}
}
@@ -0,0 +1,182 @@
import { describe, expect, it } from "vitest";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { OnboardingOrchestrator } from "./orchestrator.js";
import type { DerivedState } from "../state/onboarding-state.js";
function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
return {
phase: { kind: "photo-pending" },
intent: { kind: "help-how" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 256,
allowShortResponse: false,
tier: 0,
...overrides,
};
}
describe("OnboardingOrchestrator", () => {
it("uses direct response path for tier 0", async () => {
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
return Promise.resolve({
content: "안내 도와드릴게요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "안내 도와드릴게요." } }] },
message: { role: "assistant" as const, content: "안내 도와드릴게요." },
});
},
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ tier: 0 }),
[{ role: "user", content: "도와줘" }],
{ toolContext: {} },
);
expect(result).toBe("안내 도와드릴게요.");
expect(calls).toHaveLength(1);
expect(calls[0]?.options?.role).toBe("responder");
});
it("runs planner then responder for tier 1", async () => {
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
if (calls.length === 1) {
return Promise.resolve({
content: JSON.stringify({
approach: "redirect-to-step",
targetPhase: "photo-pending",
mustInclude: ["하트 인증만 안내"],
mustAvoid: ["디스코드 절차"],
suggestedTools: [],
tone: "friendly",
targetLengthChars: 120,
rationale: "질문 범위만 답하기",
}),
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "{}" } }] },
message: { role: "assistant" as const, content: "{}" },
});
}
return Promise.resolve({
content: "하트 인증만 남았어요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
});
},
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ tier: 1 }),
[
{ role: "system", content: "기본 시스템 프롬프트" },
{ role: "user", content: "이제 뭐 남았지" },
],
{ toolContext: {} },
);
expect(result).toBe("하트 인증만 남았어요.");
expect(calls).toHaveLength(2);
expect(calls[0]?.options?.role).toBe("planner");
expect(calls[1]?.messages[0]?.content).toContain("[PLAN]");
});
it("runs critic after responder for tier 2 and applies rewrite", async () => {
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
if (calls.length === 1) {
return Promise.resolve({
content: JSON.stringify({
approach: "answer-direct",
targetPhase: "photo-pending",
mustInclude: [],
mustAvoid: ["추가 절차 나열"],
suggestedTools: [],
tone: "concise",
targetLengthChars: 60,
rationale: "짧게 답하기",
}),
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "{}" } }] },
message: { role: "assistant" as const, content: "{}" },
});
}
if (calls.length === 2) {
return Promise.resolve({
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: {
choices: [
{ message: { content: "하트 인증만 남았어요. 커버 보기로 들어가서..." } },
],
},
message: {
role: "assistant" as const,
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
},
});
}
return Promise.resolve({
content: "하트 인증만 남았어요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
});
},
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ tier: 2 }),
[
{ role: "system", content: "기본 시스템 프롬프트" },
{ role: "user", content: "이제 뭐 남았지" },
],
{ toolContext: {} },
);
expect(result).toBe("하트 인증만 남았어요.");
expect(calls).toHaveLength(3);
expect(calls[2]?.options?.role).toBe("critic");
});
});
@@ -0,0 +1,382 @@
import { getContext } from "@obs/context";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { Tool, ToolContext } from "@tools/tool";
import {
getLlmTemperature,
getPlannerMaxTokens,
getTier0EstimatedCallMs,
getTier0MaxTokens,
getTier0Model,
getTier1EstimatedCallMs,
getTier1MaxTokens,
getTier1PlannerModel,
getTier1ResponderModel,
getTier2EstimatedCallMs,
getTier2MaxTokens,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
import { OnboardingCritic } from "./critic.js";
import type { PlannerOutput } from "./planner-schema.js";
import { PlannerOutputSchema } from "./planner-schema.js";
import { runWithTools } from "./tool-loop.js";
export interface OrchestratorRunOptions {
maxTokens?: number | undefined;
temperature?: number | undefined;
toolContext: ToolContext;
tools?: readonly Tool<unknown, unknown>[] | undefined;
}
function cloneMessages(messages: ChatMessage[]): ChatMessage[] {
return messages.map((message) => ({
...message,
tool_calls: message.tool_calls ? [...message.tool_calls] : undefined,
}));
}
function injectSystemAddendum(
messages: ChatMessage[],
addendum: string,
): ChatMessage[] {
const cloned = cloneMessages(messages);
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
if (firstSystemIndex === -1) {
cloned.unshift({
role: "system",
content: addendum,
});
return cloned;
}
const current = cloned[firstSystemIndex];
if (!current) {
return cloned;
}
cloned[firstSystemIndex] = {
...current,
content: `${current.content}\n\n${addendum}`,
};
return cloned;
}
function summarizeMessages(messages: ChatMessage[]): string {
return messages
.map((message) => `[${message.role}] ${message.content}`)
.join("\n\n");
}
function tryParsePlannerOutput(raw: string): PlannerOutput | null {
try {
const parsed = JSON.parse(raw) as unknown;
return PlannerOutputSchema.parse(parsed);
} catch {
const start = raw.indexOf("{");
const end = raw.lastIndexOf("}");
if (start === -1 || end === -1 || end <= start) {
return null;
}
try {
const parsed = JSON.parse(raw.slice(start, end + 1)) as unknown;
return PlannerOutputSchema.parse(parsed);
} catch {
return null;
}
}
}
function getDefaultPlan(derived: DerivedState): PlannerOutput {
return {
approach:
derived.responseMode === "llm-with-tools"
? "validate-then-explain"
: derived.intent.kind === "help-how" || derived.intent.kind === "help-where"
? "redirect-to-step"
: "answer-direct",
targetPhase: derived.phase.kind,
mustInclude: [],
mustAvoid: [],
suggestedTools:
derived.responseMode === "llm-with-tools" ? ["validate-nickname"] : [],
tone: derived.allowShortResponse ? "concise" : "friendly",
targetLengthChars: derived.allowShortResponse ? 80 : 150,
rationale: "fallback-plan",
};
}
function getRemainingDeadlineMs(): number | undefined {
const deadlineAt = getContext()?.deadlineAt;
if (!deadlineAt) {
return undefined;
}
const deadlineMs = Date.parse(deadlineAt) - Date.now();
return Number.isFinite(deadlineMs) ? deadlineMs : undefined;
}
function canAffordCall(estimatedCallMs: number): boolean {
const remainingMs = getRemainingDeadlineMs();
return remainingMs === undefined || remainingMs > estimatedCallMs;
}
function buildPlannerMessages(
derived: DerivedState,
messages: ChatMessage[],
): ChatMessage[] {
return [
{
role: "system",
content: [
"당신은 온보딩 응답을 설계하는 planner입니다.",
"반드시 JSON만 출력하세요.",
"질문에 직접 답할 수 있으면 answer-direct를 우선하고, 유저가 묻지 않은 추가 절차는 mustAvoid에 넣으세요.",
"tool은 꼭 필요한 경우에만 suggestedTools에 넣고 최대 3개까지만 고르세요.",
].join("\n"),
},
{
role: "user",
content: [
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}, tier=${derived.tier}`,
"[CONVERSATION]",
summarizeMessages(messages),
"[SCHEMA]",
JSON.stringify({
approach:
"answer-direct | ask-clarification | redirect-to-step | validate-then-explain | lookup-rule-then-answer | soft-decline",
targetPhase:
"fresh | nickname-pending | photo-pending | both-pending | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
mustInclude: ["short strings"],
mustAvoid: ["short strings"],
suggestedTools: [
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context",
],
tone: "friendly | concise | encouraging | neutral | apologetic",
targetLengthChars: 150,
rationale: "short reason",
}),
].join("\n\n"),
},
];
}
function buildResponderAddendum(plan: PlannerOutput): string {
const lines = [
"[PLAN]",
`approach=${plan.approach}`,
`tone=${plan.tone}`,
`targetLengthChars=${plan.targetLengthChars}`,
];
if (plan.targetPhase) {
lines.push(`targetPhase=${plan.targetPhase}`);
}
lines.push(
`mustInclude=${plan.mustInclude.length > 0 ? plan.mustInclude.join(" | ") : "-"}`,
`mustAvoid=${plan.mustAvoid.length > 0 ? plan.mustAvoid.join(" | ") : "-"}`,
"위 계획을 참고하되, 실제 유저에게 보낼 최종 한국어 답변만 출력하세요.",
);
return lines.join("\n");
}
function pickToolsForPlan(
plan: PlannerOutput,
tools: readonly Tool<unknown, unknown>[],
): readonly Tool<unknown, unknown>[] {
if (plan.approach === "answer-direct" || plan.approach === "soft-decline") {
return [];
}
if (plan.suggestedTools.length === 0) {
return tools;
}
const picked = tools.filter((tool) => plan.suggestedTools.includes(tool.name as never));
return picked.length > 0 ? picked : tools;
}
export class OnboardingOrchestrator {
private readonly critic: OnboardingCritic;
constructor(private readonly llmClient: LlmClient) {
this.critic = new OnboardingCritic(llmClient);
}
private async runDirect(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
responderTools: readonly Tool<unknown, unknown>[],
maxTokens: number,
modelOverride: string,
): Promise<string | null> {
const deadlineMs = getRemainingDeadlineMs();
if (responderTools.length > 0) {
return runWithTools({
deadlineMs,
llmClient: this.llmClient,
maxTokens,
messages,
modelOverride,
role: "responder",
tier: derived.tier,
temperature: options.temperature ?? getLlmTemperature(),
toolContext: options.toolContext,
tools: responderTools,
});
}
const response = await this.llmClient.chatCompletion(messages, {
deadlineMs,
maxTokens,
modelOverride,
role: "responder",
temperature: options.temperature ?? getLlmTemperature(),
tier: derived.tier,
});
return response?.content ?? null;
}
private async runPlanner(
derived: DerivedState,
messages: ChatMessage[],
): Promise<PlannerOutput> {
const plannerMessages = buildPlannerMessages(derived, messages);
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
deadlineMs: getRemainingDeadlineMs(),
jsonMode: true,
maxTokens: getPlannerMaxTokens(),
modelOverride: getTier1PlannerModel(),
role: "planner",
temperature: 0,
tier: 1,
});
const firstPlan = firstAttempt?.content
? tryParsePlannerOutput(firstAttempt.content)
: null;
if (firstPlan) {
return firstPlan;
}
const retryMessages = [
...plannerMessages,
{
role: "user" as const,
content:
"방금 응답은 스키마를 벗어났어요. 설명 없이 JSON object 하나만 다시 출력하세요.",
},
];
const retry = await this.llmClient.chatCompletion(retryMessages, {
deadlineMs: getRemainingDeadlineMs(),
jsonMode: true,
maxTokens: getPlannerMaxTokens(),
modelOverride: getTier1PlannerModel(),
role: "planner",
temperature: 0,
tier: 1,
});
const retryPlan = retry?.content ? tryParsePlannerOutput(retry.content) : null;
return retryPlan ?? getDefaultPlan(derived);
}
private async runTier0(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
const maxTokens = options.maxTokens ?? getTier0MaxTokens();
const tools =
derived.responseMode === "llm-with-tools" ? (options.tools ?? []) : [];
return this.runDirect(
derived,
messages,
options,
tools,
maxTokens,
getTier0Model(),
);
}
private async runTier1(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
const plan = await this.runPlanner(derived, messages);
const responderMessages = injectSystemAddendum(
messages,
buildResponderAddendum(plan),
);
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
return this.runDirect(
derived,
responderMessages,
options,
responderTools,
options.maxTokens ?? getTier1MaxTokens(),
getTier1ResponderModel(),
);
}
private async runTier2(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
const plan = await this.runPlanner(derived, messages);
const responderMessages = injectSystemAddendum(
messages,
buildResponderAddendum(plan),
);
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
const draft = await this.runDirect(
derived,
responderMessages,
options,
responderTools,
options.maxTokens ?? getTier2MaxTokens(),
getTier1ResponderModel(),
);
if (!draft) {
return null;
}
const critique = await this.critic.critique(draft, derived, responderMessages, {
deadlineMs: getRemainingDeadlineMs(),
});
return critique.text;
}
async run(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
if (derived.tier >= 2 && canAffordCall(getTier2EstimatedCallMs())) {
return this.runTier2(derived, messages, options);
}
if (derived.tier >= 1 && canAffordCall(getTier1EstimatedCallMs())) {
return this.runTier1(derived, messages, options);
}
if (!canAffordCall(getTier0EstimatedCallMs())) {
return null;
}
return this.runTier0(derived, messages, options);
}
}
@@ -0,0 +1,50 @@
import { z } from "zod";
export const PlannerApproachSchema = z.enum([
"answer-direct",
"ask-clarification",
"redirect-to-step",
"validate-then-explain",
"lookup-rule-then-answer",
"soft-decline",
]);
export const PlannerTargetPhaseSchema = z.enum([
"fresh",
"nickname-pending",
"photo-pending",
"both-pending",
"nickname-rejected",
"photo-rejected",
"deadline-near",
"completed",
"timed-out",
]);
export const PlannerSuggestedToolSchema = z.enum([
"validate-nickname",
"check-onboarding-progress",
"lookup-community-rule",
"get-room-context",
]);
export const PlannerToneSchema = z.enum([
"friendly",
"concise",
"encouraging",
"neutral",
"apologetic",
]);
export const PlannerOutputSchema = z.object({
approach: PlannerApproachSchema,
targetPhase: PlannerTargetPhaseSchema.optional(),
mustInclude: z.array(z.string().min(1).max(80)).max(5).default([]),
mustAvoid: z.array(z.string().min(1).max(80)).max(5).default([]),
suggestedTools: z.array(PlannerSuggestedToolSchema).max(3).default([]),
tone: PlannerToneSchema.default("friendly"),
targetLengthChars: z.number().int().min(20).max(500).default(150),
rationale: z.string().max(200),
});
export type PlannerOutput = z.infer<typeof PlannerOutputSchema>;
@@ -0,0 +1,219 @@
import { describe, expect, it } from "vitest";
import { z } from "zod";
import { runWithTools } from "./tool-loop.js";
import type { Tool } from "@tools/tool";
describe("runWithTools", () => {
it("executes a tool call and returns the model's final answer", async () => {
const calls: Array<{
content: string;
messages: Array<{ role: string; content: string }>;
options?: Record<string, unknown> | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: Array<{ role: string; content: string }>,
options?: Record<string, unknown>,
) => {
calls.push({
content: messages[messages.length - 1]?.content ?? "",
messages,
options,
});
if (calls.length === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: {
choices: [
{
message: {
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
},
],
},
message: {
role: "assistant",
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
endpoint: "mock://llm",
provider: "llama",
raw: {
choices: [
{
message: {
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
},
},
],
},
message: {
role: "assistant",
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
},
});
},
};
const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = {
name: "validate-nickname",
description: "닉네임 형식 검사",
argsSchema: z.object({
nickname: z.string(),
}),
resultSchema: z.object({
valid: z.boolean(),
issues: z.array(z.string()),
}),
execute: () =>
Promise.resolve({
valid: false,
issues: ["티어는 '/'로 구분해서 넣어주세요."],
}),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [
{
role: "system",
content: "도구를 필요할 때만 사용해.",
},
{
role: "user",
content: "닉네임이 왜 안 돼요?",
},
],
toolContext: {},
tools: [tool],
});
expect(result).toContain("프리리/95/여/D");
expect(calls[0]?.options?.tools).toHaveLength(1);
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
});
it("preserves native tool call responses for follow-up turns", async () => {
const seenMessages: Array<Array<Record<string, unknown>>> = [];
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
seenMessages.push(messages);
if (seenMessages.length === 1) {
return Promise.resolve({
content: "",
endpoint: "mock://cliproxy",
provider: "cliproxy",
raw: {
choices: [
{
message: {
content: null,
tool_calls: [
{
id: "call_1",
type: "function",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
},
},
],
},
message: {
role: "assistant",
content: "",
tool_calls: [
{
id: "call_1",
type: "function",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
},
});
}
return Promise.resolve({
content: "나이 기준은 27세 이상이에요.",
endpoint: "mock://cliproxy",
provider: "cliproxy",
raw: {
choices: [
{
message: {
content: "나이 기준은 27세 이상이에요.",
},
},
],
},
message: {
role: "assistant",
content: "나이 기준은 27세 이상이에요.",
},
});
},
};
const tool: Tool<{ topic: string }, { topic: string; text: string }> = {
name: "lookup-community-rule",
description: "커뮤니티 규칙 조회",
argsSchema: z.object({
topic: z.string(),
}),
resultSchema: z.object({
topic: z.string(),
text: z.string(),
}),
execute: () =>
Promise.resolve({
topic: "age",
text: "나이 기준은 27세 이상이에요.",
}),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [
{
role: "system",
content: "필요할 때만 도구를 사용해.",
},
{
role: "user",
content: "나이 기준이 뭐예요?",
},
],
toolContext: {},
tools: [tool],
});
expect(result).toBe("나이 기준은 27세 이상이에요.");
expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true);
expect(
seenMessages[1]?.some(
(message) => Array.isArray(message.tool_calls) && message.role === "assistant",
),
).toBe(true);
});
});
@@ -0,0 +1,89 @@
import { withToolSpan } from "@obs/spans";
import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
import type { Tool, ToolContext } from "@tools/tool";
export interface ToolLoopOptions {
deadlineMs?: number | undefined;
llmClient: LlmClient;
maxHops?: number | undefined;
maxTokens?: number | undefined;
messages: ChatMessage[];
modelOverride?: string | undefined;
role?: ChatCompletionOptions["role"];
signal?: AbortSignal | undefined;
temperature?: number | undefined;
tier?: ChatCompletionOptions["tier"];
toolContext: ToolContext;
tools: readonly Tool<unknown, unknown>[];
}
export async function runWithTools(
options: ToolLoopOptions,
): Promise<string | null> {
const maxHops = options.maxHops ?? 2;
const messages = options.messages.map((message) => ({ ...message }));
let allowNativeTools = true;
for (let hop = 0; hop <= maxHops; hop++) {
const response = await options.llmClient.chatCompletion(messages, {
deadlineMs: options.deadlineMs,
maxTokens: options.maxTokens,
modelOverride: options.modelOverride,
role: options.role,
signal: options.signal,
temperature: options.temperature,
tier: options.tier,
tools: allowNativeTools ? options.tools : undefined,
});
if (!response) {
return null;
}
const toolCalls = parseToolCalls(response.message);
if (toolCalls.length === 0) {
return response.content;
}
if (hop === maxHops) {
messages.push(response.message);
messages.push({
role: "user",
content:
"도구 호출은 여기까지 하고, 지금까지 얻은 정보만으로 한국어 문장으로 최종 답변해줘.",
});
allowNativeTools = false;
continue;
}
messages.push(response.message);
for (const call of toolCalls) {
const tool = options.tools.find((candidate) => candidate.name === call.name);
if (!tool) {
throw new Error(`Tool not registered for call: ${call.name}`);
}
const validated = tool.argsSchema.parse(call.arguments);
const result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
messages.push({
role: "tool",
name: tool.name,
tool_call_id: call.id,
content: JSON.stringify(result),
});
}
}
return null;
}
@@ -0,0 +1,67 @@
import { describe, expect, it } from "vitest";
import { validateOnboardingResponse } from "./validator.js";
import type { DerivedState } from "../state/onboarding-state.js";
function makeDerived(
overrides?: Partial<DerivedState>,
): DerivedState {
return {
phase: { kind: "photo-pending" },
intent: { kind: "photo-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 256,
allowShortResponse: false,
tier: 0,
...overrides,
};
}
describe("validateOnboardingResponse", () => {
it("rejects discord guidance before onboarding is completed", () => {
const result = validateOnboardingResponse(
"디스코드 서버에 들어가시면 돼요.",
makeDerived(),
);
expect(result.status).toBe("reject");
expect(result.reasons).toContain("stateConsistency");
});
it("rejects invalid nickname examples", () => {
const result = validateOnboardingResponse(
"홍길동/95/남/Z로 바꾸시면 돼요.",
makeDerived({
phase: { kind: "nickname-rejected", current: "홍길동/95/남/골드", issues: [] },
intent: { kind: "nickname-question" },
}),
);
expect(result.status).toBe("reject");
expect(result.reasons).toContain("nicknameExampleConsistency");
});
it("soft-warns on repeated phrases", () => {
const result = validateOnboardingResponse(
"하트 인증 해주세요 하트 인증 해주세요 하트 인증 해주세요",
makeDerived(),
{ allowShortResponse: true },
);
expect(result.status).toBe("soft-warn");
expect(result.reasons).toContain("phraseRepetition");
});
it("passes short status responses when allowShortResponse is enabled", () => {
const result = validateOnboardingResponse(
"이제 방 하트 인증만 남았어요.",
makeDerived({
intent: { kind: "status-question" },
}),
{ allowShortResponse: true },
);
expect(result.status).toBe("pass");
});
});
+180
View File
@@ -0,0 +1,180 @@
import { trace } from "@opentelemetry/api";
import { getMaxResponseLength } from "../../../config.js";
import { VALID_TIER_SET } from "../constants.js";
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
import type { DerivedState } from "../state/onboarding-state.js";
const ALLOWED_URL_PATTERNS = [
/discord\.gg\/lolympus/iu,
/open\.kakao\.com\/o\/gOscBAEh/iu,
];
export interface ValidationResult {
status: "pass" | "soft-warn" | "reject";
text: string;
reasons: string[];
}
function postProcessResponse(text: string): string {
let processed = text.trim();
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
processed = processed.replace(
/https?:\/\/[^\s)}\]]+/gu,
(match) => {
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
return match;
}
return "";
},
);
processed = processed.replace(/ +/gu, " ").trim();
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1");
processed = processed.replace(/\*(.+?)\*/gu, "$1");
processed = processed.replace(/__(.+?)__/gu, "$1");
processed = processed.replace(/~~(.+?)~~/gu, "$1");
processed = processed.replace(/^#{1,6}\s+/gmu, "");
processed = processed.replace(/^[-*]\s+/gmu, "");
processed = processed.replace(/`(.+?)`/gu, "$1");
const maxLen = getMaxResponseLength();
if (processed.length > maxLen) {
processed = processed.slice(0, maxLen).trimEnd();
const sentenceEndPattern =
/(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
for (let index = processed.length; index > maxLen * 0.5; index--) {
const sub = processed.slice(0, index);
if (sentenceEndPattern.test(sub)) {
processed = sub;
break;
}
}
}
return processed.trim();
}
function findInvalidNicknameExamples(text: string): string[] {
const matches = [
...text.matchAll(/([^\s/]{1,20})\/(\d{2})\/(남|여)\/([A-Z]{1,2})/gu),
];
return matches.flatMap((match) => {
const tier = match[4];
if (!tier || VALID_TIER_SET.has(tier)) {
return [];
}
return [tier];
});
}
function hasIntentKeywordMatch(text: string, derived: DerivedState): boolean {
const keywords = EXPECTED_KEYWORDS[derived.intent.kind];
if (keywords.length === 0) {
return true;
}
const lower = text.toLowerCase();
return keywords.some((keyword) => lower.includes(keyword.toLowerCase()));
}
function hasRepeatedPhrase(text: string): boolean {
const words = text.split(/\s+/u);
if (words.length < 6) {
return false;
}
const phrases = new Map<string, number>();
for (let index = 0; index < words.length - 1; index++) {
const bigram = `${words[index]} ${words[index + 1]}`;
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
}
return [...phrases.values()].some((count) => count >= 3);
}
function markValidation(result: ValidationResult): ValidationResult {
const activeSpan = trace.getActiveSpan();
activeSpan?.setAttributes({
"agent.validator.status": result.status,
"agent.validator.warnings": result.reasons,
});
return result;
}
export function validateOnboardingResponse(
draft: string,
derived: DerivedState,
options?: { allowShortResponse?: boolean | undefined },
): ValidationResult {
const text = postProcessResponse(draft);
const rejectReasons: string[] = [];
const softWarnReasons: string[] = [];
if (!text) {
rejectReasons.push("empty");
}
if (text.includes("[NO_RESPONSE]")) {
return markValidation({
status: "pass",
text: "[NO_RESPONSE]",
reasons: [],
});
}
if (!options?.allowShortResponse && text.length < 10) {
rejectReasons.push("minLength");
}
const mentionsDiscord = /(디스코드|discord|디코)/iu.test(text);
const advancesDiscordStep = /(서버|프로필|권한|입장)/iu.test(text);
const defersDiscordStep =
/(먼저|완료.*후|완료하신 뒤|완료하신 후|모두 완료|완료되면)/u.test(text);
if (
mentionsDiscord &&
derived.phase.kind !== "completed" &&
advancesDiscordStep &&
!defersDiscordStep
) {
rejectReasons.push("stateConsistency");
}
if (findInvalidNicknameExamples(text).length > 0) {
rejectReasons.push("nicknameExampleConsistency");
}
if (!hasIntentKeywordMatch(text, derived)) {
softWarnReasons.push("intentAlignment");
}
if (hasRepeatedPhrase(text)) {
softWarnReasons.push("phraseRepetition");
}
if (rejectReasons.length > 0) {
return markValidation({
status: "reject",
text,
reasons: [...rejectReasons, ...softWarnReasons],
});
}
if (softWarnReasons.length > 0) {
return markValidation({
status: "soft-warn",
text,
reasons: softWarnReasons,
});
}
return markValidation({
status: "pass",
text,
reasons: [],
});
}