Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

@@ -0,0 +1,64 @@
import { z } from "zod";
import { getOnboardingSessionKeyPrefix } from "../../config.js";
import type { OnboardingSession } from "@skills/onboarding/types";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
chatroomId: z.string().min(1),
userId: z.string().min(1),
});
const resultSchema = z.object({
nicknameDone: z.boolean(),
photoDone: z.boolean(),
remainingSteps: z.array(z.enum(["nickname", "photo"])),
deadlineInHours: z.number().nullable(),
});
function getRemainingSteps(session: OnboardingSession | null): Array<"nickname" | "photo"> {
if (!session) {
return ["nickname", "photo"];
}
const remaining: Array<"nickname" | "photo"> = [];
if (!session.nicknameVerified) {
remaining.push("nickname");
}
if (!session.photoVerified) {
remaining.push("photo");
}
return remaining;
}
export const checkOnboardingProgressTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "check-onboarding-progress",
description:
"현재 유저의 온보딩 진행 상태를 읽어 남은 절차와 데드라인까지 남은 시간을 구조화해서 반환한다.",
argsSchema,
resultSchema,
async execute(args, ctx) {
const session =
ctx.session && ctx.chatroomId === args.chatroomId && ctx.userId === args.userId
? ctx.session
: ctx.garnetClient
? await ctx.garnetClient.getJson<OnboardingSession>(
`${getOnboardingSessionKeyPrefix()}:onboarding:${args.chatroomId}:${args.userId}`,
)
: null;
const deadlineAt = session?.deadlineAt ? Date.parse(session.deadlineAt) : NaN;
const deadlineInHours = Number.isFinite(deadlineAt)
? Math.max(0, (deadlineAt - Date.now()) / 3_600_000)
: null;
return {
nicknameDone: session?.nicknameVerified ?? false,
photoDone: session?.photoVerified ?? false,
remainingSteps: getRemainingSteps(session),
deadlineInHours,
};
},
};
+48
View File
@@ -0,0 +1,48 @@
import { z } from "zod";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
chatroomId: z.string().min(1),
limit: z.number().int().min(1).max(20).optional(),
});
const roomMessageSchema = z.object({
senderId: z.string(),
nickname: z.string(),
message: z.string(),
timestamp: z.string(),
});
const resultSchema = z.object({
messages: z.array(roomMessageSchema),
});
export const getRoomContextTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "get-room-context",
description:
"최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.",
argsSchema,
resultSchema,
async execute(args, ctx) {
if (!ctx.roomContextService) {
return { messages: [] };
}
const messages = await ctx.roomContextService.getRecentMessages(
args.chatroomId,
args.limit,
);
return {
messages: messages.map((message) => ({
senderId: message.senderId,
nickname: message.nickname,
message: message.message,
timestamp: message.timestamp.toISOString(),
})),
};
},
};
+20
View File
@@ -0,0 +1,20 @@
import { ToolRegistry } from "../registry.js";
import { checkOnboardingProgressTool } from "./check-onboarding-progress.js";
import { getRoomContextTool } from "./get-room-context.js";
import { lookupCommunityRuleTool } from "./lookup-community-rule.js";
import { validateNicknameTool } from "./validate-nickname.js";
export function createBuiltInToolRegistry(): ToolRegistry {
return new ToolRegistry()
.register(validateNicknameTool)
.register(checkOnboardingProgressTool)
.register(lookupCommunityRuleTool)
.register(getRoomContextTool);
}
export {
checkOnboardingProgressTool,
getRoomContextTool,
lookupCommunityRuleTool,
validateNicknameTool,
};
@@ -0,0 +1,54 @@
import { z } from "zod";
import {
DEADLINE_NEAR_THRESHOLD_MIN,
DISCORD_GUIDE_TEXT,
HEART_RULE_TEXT,
MIN_AGE,
NICKNAME_FORMAT_TEXT,
TIER_CODES_TEXT,
} from "@skills/onboarding/constants";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
topic: z.enum([
"age",
"tier-codes",
"discord-link",
"heart-rule",
"nickname-format",
"deadline",
]),
});
const resultSchema = z.object({
topic: argsSchema.shape.topic,
text: z.string(),
source: z.literal("constants"),
});
const TOPIC_TEXT: Record<z.infer<typeof argsSchema>["topic"], string> = {
age: `가입 가능 연령은 ${MIN_AGE}세 이상이에요.`,
"tier-codes": `사용 가능한 티어 코드는 ${TIER_CODES_TEXT}예요.`,
"discord-link": DISCORD_GUIDE_TEXT,
"heart-rule": HEART_RULE_TEXT,
"nickname-format": NICKNAME_FORMAT_TEXT,
deadline: `데드라인이 ${DEADLINE_NEAR_THRESHOLD_MIN}분 이하로 남으면 마감이 임박한 상태로 안내해요.`,
};
export const lookupCommunityRuleTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "lookup-community-rule",
description:
"나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 상수에서 조회한다.",
argsSchema,
resultSchema,
execute(args) {
return Promise.resolve({
topic: args.topic,
text: TOPIC_TEXT[args.topic],
source: "constants",
});
},
};
+52
View File
@@ -0,0 +1,52 @@
import { z } from "zod";
import { analyzeNicknameIssues } from "@skills/onboarding/onboarding-templates";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
nickname: z.string().min(1),
});
const nicknameIssueSchema = z.object({
kind: z.enum([
"missing-tier",
"invalid-age",
"missing-gender",
"wrong-separator",
"unknown-tier",
"other",
]),
message: z.string(),
suggestion: z.string().optional(),
});
const resultSchema = z.object({
valid: z.boolean(),
issues: z.array(nicknameIssueSchema),
normalized: z.string().optional(),
});
export const validateNicknameTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "validate-nickname",
description:
"새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.",
argsSchema,
resultSchema,
execute(args) {
const normalized = args.nickname.trim();
const issues = analyzeNicknameIssues(normalized);
const result: z.infer<typeof resultSchema> = {
valid: issues.length === 0,
issues,
};
if (issues.length === 0) {
result.normalized = normalized;
}
return Promise.resolve(result);
},
};
+8
View File
@@ -0,0 +1,8 @@
export { ToolRegistry } from "./registry.js";
export { parseToolCalls } from "./parser.js";
export {
serializeToolsForHermes,
serializeToolsForOpenAi,
} from "./serializer.js";
export type { OpenAiToolDefinition } from "./serializer.js";
export type { Tool, ToolContext } from "./tool.js";
+70
View File
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import { parseToolCalls } from "./parser.js";
describe("parseToolCalls", () => {
it("parses Hermes tool call blocks", () => {
const result = parseToolCalls(
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여/D"}}</tool_call>',
);
expect(result).toEqual([
{
name: "validate-nickname",
arguments: {
nickname: "프리리/95/여/D",
},
},
]);
});
it("parses OpenAI tool_calls arrays", () => {
const result = parseToolCalls({
tool_calls: [
{
id: "call_1",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
});
expect(result).toEqual([
{
id: "call_1",
name: "lookup-community-rule",
arguments: {
topic: "age",
},
},
]);
});
it("parses tool calls nested in choices[0].message", () => {
const result = parseToolCalls({
choices: [
{
message: {
tool_calls: [
{
id: "call_2",
function: {
name: "check-onboarding-progress",
arguments: '{"chatroomId":"room-1","userId":"user-1"}',
},
},
],
},
},
],
});
expect(result[0]?.name).toBe("check-onboarding-progress");
expect(result[0]?.arguments).toEqual({
chatroomId: "room-1",
userId: "user-1",
});
});
});
+136
View File
@@ -0,0 +1,136 @@
export interface ParsedToolCall {
id?: string | undefined;
name: string;
arguments: Record<string, unknown>;
}
interface OpenAiToolCallShape {
id?: string;
function?: {
name?: string;
arguments?: string;
};
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function parseJsonArguments(raw: unknown): Record<string, unknown> | null {
if (typeof raw === "string") {
try {
const parsed = JSON.parse(raw) as unknown;
return asRecord(parsed);
} catch {
return null;
}
}
if (typeof raw === "object" && raw !== null) {
return raw as Record<string, unknown>;
}
return null;
}
function parseHermesCalls(content: string): ParsedToolCall[] {
const matches = [...content.matchAll(/<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/gu)];
return matches.flatMap((match) => {
const payload = match[1];
if (!payload) {
return [];
}
try {
const parsed = JSON.parse(payload) as
| {
id?: string;
name?: string;
arguments?: unknown;
}
| Array<{
id?: string;
name?: string;
arguments?: unknown;
}>;
const items = Array.isArray(parsed) ? parsed : [parsed];
return items.flatMap((item) => {
const args = parseJsonArguments(item.arguments);
if (!item.name || !args) {
return [];
}
return [
{
id: item.id,
name: item.name,
arguments: args,
},
];
});
} catch {
return [];
}
});
}
function parseOpenAiCalls(value: unknown): ParsedToolCall[] {
const record = asRecord(value);
const choices = Array.isArray(record["choices"])
? (record["choices"] as unknown[])
: [];
const firstChoice = choices[0];
const candidates = [
record["tool_calls"],
asRecord(record["message"])["tool_calls"],
asRecord(asRecord(firstChoice)["message"])["tool_calls"],
];
const toolCallsCandidate = candidates.find(Array.isArray);
if (!Array.isArray(toolCallsCandidate)) {
return [];
}
return toolCallsCandidate.flatMap((item) => {
const call = item as OpenAiToolCallShape;
const args = parseJsonArguments(call.function?.arguments);
const name = call.function?.name;
if (!name || !args) {
return [];
}
return [
{
id: call.id,
name,
arguments: args,
},
];
});
}
export function parseToolCalls(value: unknown): ParsedToolCall[] {
if (typeof value === "string") {
return parseHermesCalls(value);
}
const record = asRecord(value);
const content =
typeof record["content"] === "string"
? record["content"]
: typeof asRecord(record["message"])["content"] === "string"
? (asRecord(record["message"])["content"] as string)
: "";
const parsedOpenAiCalls = parseOpenAiCalls(value);
if (parsedOpenAiCalls.length > 0) {
return parsedOpenAiCalls;
}
return content ? parseHermesCalls(content) : [];
}
+32
View File
@@ -0,0 +1,32 @@
import type { Tool } from "./tool.js";
export class ToolRegistry {
private readonly tools = new Map<string, Tool<unknown, unknown>>();
register<A, R>(tool: Tool<A, R>): this {
this.tools.set(tool.name, tool as Tool<unknown, unknown>);
return this;
}
get(name: string): Tool<unknown, unknown> {
const tool = this.tools.get(name);
if (!tool) {
throw new Error(`Unknown tool: ${name}`);
}
return tool;
}
has(name: string): boolean {
return this.tools.has(name);
}
list(): Tool<unknown, unknown>[] {
return [...this.tools.values()];
}
pick(names: readonly string[]): Tool<unknown, unknown>[] {
return names
.filter((name) => this.tools.has(name))
.map((name) => this.get(name));
}
}
+42
View File
@@ -0,0 +1,42 @@
import { z } from "zod";
import type { Tool } from "./tool.js";
export interface OpenAiToolDefinition {
type: "function";
function: {
name: string;
description: string;
parameters: unknown;
};
}
function toToolDescriptor(tool: Tool<unknown, unknown>): {
name: string;
description: string;
parameters: unknown;
} {
return {
name: tool.name,
description: tool.description,
parameters: z.toJSONSchema(tool.argsSchema),
};
}
export function serializeToolsForHermes(
tools: readonly Tool<unknown, unknown>[],
): string {
const descriptors = tools.map(toToolDescriptor);
return `<tools>\n${JSON.stringify(descriptors, null, 2)}\n</tools>`;
}
export function serializeToolsForOpenAi(
tools: readonly Tool<unknown, unknown>[],
): OpenAiToolDefinition[] {
return tools.map((tool) => {
const descriptor = toToolDescriptor(tool);
return {
type: "function",
function: descriptor,
};
});
}
+20
View File
@@ -0,0 +1,20 @@
import type { GarnetClient } from "@common/garnet";
import type { RoomContextService } from "@room/room-context-service";
import type { ZodType } from "zod";
import type { OnboardingSession } from "@skills/onboarding/types";
export interface ToolContext {
chatroomId?: string | undefined;
userId?: string | undefined;
session?: OnboardingSession | null | undefined;
garnetClient?: GarnetClient | undefined;
roomContextService?: RoomContextService | undefined;
}
export interface Tool<A, R> {
name: string;
description: string;
argsSchema: ZodType<A>;
resultSchema: ZodType<R>;
execute(args: A, ctx: ToolContext): Promise<R>;
}