From 5579db54da940fe6415400743f856d8710fd84c6 Mon Sep 17 00:00:00 2001 From: dahyeon Date: Fri, 17 Apr 2026 00:47:02 -0700 Subject: [PATCH] Add eval runner and tool orchestration --- ecosystem.config.cjs | 18 + launch-server.ps1 | 2 +- package.json | 3 +- src/agent/agent.ts | 9 +- src/common/garnet.ts | 232 +++++ src/common/nats-subjects.ts | 19 + src/config.ts | 148 ++- src/conversation/conversation-store.ts | 30 +- src/conversation/summarizer.ts | 2 +- src/eval/cli.ts | 183 +++- src/eval/golden/onboarding.jsonl | 3 + src/eval/judges/llm-judge.test.ts | 109 +++ src/eval/judges/llm-judge.ts | 134 +++ src/eval/judges/rule-judge.test.ts | 70 ++ src/eval/judges/rule-judge.ts | 121 +++ src/eval/runner.test.ts | 56 ++ src/eval/runner.ts | 842 ++++++++++++++++++ src/eval/shadow-bus.test.ts | 93 ++ src/eval/shadow-bus.ts | 90 ++ src/eval/shadow-worker.test.ts | 53 ++ src/eval/shadow-worker.ts | 741 +++++++++++++++ src/eval/types.ts | 151 ++++ src/llm/index.ts | 3 + src/llm/llm-client.ts | 147 +-- src/llm/providers/cliproxy.ts | 30 + src/llm/providers/index.ts | 18 + src/llm/providers/llama.ts | 32 + src/llm/providers/openrouter.ts | 21 + src/llm/providers/shared.ts | 235 +++++ src/llm/providers/types.ts | 33 + src/llm/routing.test.ts | 102 +++ src/llm/routing.ts | 99 ++ src/llm/types.ts | 47 +- src/main.ts | 44 +- src/observability/alert.test.ts | 40 + src/observability/alert.ts | 66 ++ src/observability/tracing.ts | 11 +- src/skills/onboarding/constants.ts | 45 + src/skills/onboarding/onboarding-templates.ts | 35 +- src/skills/onboarding/prompts/builder.ts | 8 +- src/skills/onboarding/prompts/layers.ts | 14 +- src/skills/onboarding/prompts/tool-policy.ts | 28 + src/skills/onboarding/runtime/critic.ts | 83 ++ .../onboarding/runtime/orchestrator.test.ts | 182 ++++ src/skills/onboarding/runtime/orchestrator.ts | 382 ++++++++ .../onboarding/runtime/planner-schema.ts | 50 ++ .../onboarding/runtime/tool-loop.test.ts | 219 +++++ src/skills/onboarding/runtime/tool-loop.ts | 89 ++ .../onboarding/runtime/validator.test.ts | 67 ++ src/skills/onboarding/runtime/validator.ts | 180 ++++ src/skills/onboarding/skill.ts | 191 ++-- .../onboarding/state/intent-keywords.ts | 18 +- src/skills/onboarding/state/transitions.ts | 4 +- .../builtins/check-onboarding-progress.ts | 64 ++ src/tools/builtins/get-room-context.ts | 48 + src/tools/builtins/index.ts | 20 + src/tools/builtins/lookup-community-rule.ts | 54 ++ src/tools/builtins/validate-nickname.ts | 52 ++ src/tools/index.ts | 8 + src/tools/parser.test.ts | 70 ++ src/tools/parser.ts | 136 +++ src/tools/registry.ts | 32 + src/tools/serializer.ts | 42 + src/tools/tool.ts | 20 + src/workers/background.ts | 207 +++++ tsconfig.alias.json | 19 + 66 files changed, 6162 insertions(+), 242 deletions(-) create mode 100644 ecosystem.config.cjs create mode 100644 src/common/nats-subjects.ts create mode 100644 src/eval/golden/onboarding.jsonl create mode 100644 src/eval/judges/llm-judge.test.ts create mode 100644 src/eval/judges/llm-judge.ts create mode 100644 src/eval/judges/rule-judge.test.ts create mode 100644 src/eval/judges/rule-judge.ts create mode 100644 src/eval/runner.test.ts create mode 100644 src/eval/runner.ts create mode 100644 src/eval/shadow-bus.test.ts create mode 100644 src/eval/shadow-bus.ts create mode 100644 src/eval/shadow-worker.test.ts create mode 100644 src/eval/shadow-worker.ts create mode 100644 src/eval/types.ts create mode 100644 src/llm/providers/cliproxy.ts create mode 100644 src/llm/providers/index.ts create mode 100644 src/llm/providers/llama.ts create mode 100644 src/llm/providers/openrouter.ts create mode 100644 src/llm/providers/shared.ts create mode 100644 src/llm/providers/types.ts create mode 100644 src/llm/routing.test.ts create mode 100644 src/llm/routing.ts create mode 100644 src/observability/alert.test.ts create mode 100644 src/observability/alert.ts create mode 100644 src/skills/onboarding/constants.ts create mode 100644 src/skills/onboarding/prompts/tool-policy.ts create mode 100644 src/skills/onboarding/runtime/critic.ts create mode 100644 src/skills/onboarding/runtime/orchestrator.test.ts create mode 100644 src/skills/onboarding/runtime/orchestrator.ts create mode 100644 src/skills/onboarding/runtime/planner-schema.ts create mode 100644 src/skills/onboarding/runtime/tool-loop.test.ts create mode 100644 src/skills/onboarding/runtime/tool-loop.ts create mode 100644 src/skills/onboarding/runtime/validator.test.ts create mode 100644 src/skills/onboarding/runtime/validator.ts create mode 100644 src/tools/builtins/check-onboarding-progress.ts create mode 100644 src/tools/builtins/get-room-context.ts create mode 100644 src/tools/builtins/index.ts create mode 100644 src/tools/builtins/lookup-community-rule.ts create mode 100644 src/tools/builtins/validate-nickname.ts create mode 100644 src/tools/index.ts create mode 100644 src/tools/parser.test.ts create mode 100644 src/tools/parser.ts create mode 100644 src/tools/registry.ts create mode 100644 src/tools/serializer.ts create mode 100644 src/tools/tool.ts create mode 100644 src/workers/background.ts create mode 100644 tsconfig.alias.json diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs new file mode 100644 index 0000000..57b8a36 --- /dev/null +++ b/ecosystem.config.cjs @@ -0,0 +1,18 @@ +module.exports = { + apps: [ + { + name: "agent", + script: "dist/main.js", + instances: 1, + max_memory_restart: "512M", + kill_timeout: 12000, + }, + { + name: "agent-worker", + script: "dist/workers/background.js", + instances: 1, + max_memory_restart: "512M", + kill_timeout: 12000, + }, + ], +}; diff --git a/launch-server.ps1 b/launch-server.ps1 index 8ce6e21..1b5f27e 100644 --- a/launch-server.ps1 +++ b/launch-server.ps1 @@ -65,7 +65,7 @@ Get-ChildItem Env: | Where-Object { $_.Name -like "*AWS*" -or $_.Name -eq "NODE_ if ($UsePm2) { - pm2 start dist/main.js --name $Pm2Name --update-env + pm2 start ecosystem.config.cjs --only agent,agent-worker --update-env } else { node ./dist/main.js } diff --git a/package.json b/package.json index 91279dc..32d3315 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "main": "dist/main.js", "type": "module", "scripts": { - "build": "tsc && tsc-alias", + "build": "tsc && tsc-alias -p tsconfig.alias.json --resolve-full-paths --resolve-full-extension .js", "typecheck": "tsc --noEmit", "test": "vitest run", "test:watch": "vitest", @@ -14,6 +14,7 @@ "format": "prettier --write .", "eval": "node --import tsx src/eval/cli.ts run --set=golden", "eval:capture": "node --import tsx src/eval/cli.ts capture", + "eval:baseline": "node --import tsx src/eval/cli.ts baseline", "start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"", "start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"", "start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"", diff --git a/src/agent/agent.ts b/src/agent/agent.ts index 7a0e26b..c482c09 100644 --- a/src/agent/agent.ts +++ b/src/agent/agent.ts @@ -6,7 +6,11 @@ import { getLogger } from "@obs/logger"; import { withSkillSpan, withSpan } from "@obs/spans"; import type { Skill } from "@skills/skill"; import type { AgentRequest, AgentResponse } from "./types.js"; -import { getRequestSubject } from "../config.js"; +import { + getAgentRequestTimeoutMs, + getRequestSubject, + getSafetyMarginMs, +} from "../config.js"; function isValidRequest(data: unknown): data is AgentRequest { if (typeof data !== "object" || data === null) return false; @@ -64,6 +68,9 @@ export class Agent { await runWithContext( { chatroomId: request.chatroomId, + deadlineAt: new Date( + Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(), + ).toISOString(), requestId: request.requestId, skill: request.skill, startedAt: new Date().toISOString(), diff --git a/src/common/garnet.ts b/src/common/garnet.ts index 99d674e..16263fc 100644 --- a/src/common/garnet.ts +++ b/src/common/garnet.ts @@ -4,6 +4,37 @@ import { getLogger } from "@obs/logger"; type GarnetRedisClient = ReturnType; +export interface StreamEntry { + deliveriesCounter?: number | undefined; + id: string; + message: Record; + millisElapsedFromDelivery?: number | undefined; +} + +export interface StreamReadResult { + messages: StreamEntry[]; + name: string; +} + +export interface StreamPendingEntry { + consumer: string; + deliveriesCounter: number; + id: string; + millisecondsSinceLastDelivery: number; +} + +interface RawStreamMessage { + deliveriesCounter?: number | undefined; + id: string; + message: unknown; + millisElapsedFromDelivery?: number | undefined; +} + +interface RawStreamReadResult { + messages: RawStreamMessage[]; + name: string; +} + function normalizeGarnetUrl(endpoint: string): string { const trimmed = endpoint.trim(); if (!trimmed) { @@ -17,6 +48,29 @@ function normalizeGarnetUrl(endpoint: string): string { return `redis://${trimmed}`; } +function normalizeStreamFields( + fields: unknown, +): Record { + if (fields instanceof Map) { + return Object.fromEntries( + [...fields.entries()].map(([key, value]) => [ + String(key), + String(value), + ]), + ); + } + + if (typeof fields === "object" && fields !== null) { + return Object.fromEntries( + Object.entries(fields as Record).map( + ([key, value]) => [key, String(value)], + ), + ); + } + + return {}; +} + export class GarnetClient { private client: GarnetRedisClient | null = null; private readonly logger = getLogger(); @@ -98,6 +152,27 @@ export class GarnetClient { await client.set(key, value); } + async setStringIfAbsent( + key: string, + value: string, + ttlSeconds?: number, + ): Promise { + const client = this.getClient(); + const setOptions: { + EX?: number; + NX: true; + } = { + NX: true, + }; + if (ttlSeconds && ttlSeconds > 0) { + setOptions.EX = ttlSeconds; + } + const result = await client.set(key, value, { + ...setOptions, + }); + return result === "OK"; + } + async delete(key: string): Promise { return this.getClient().del(key); } @@ -132,6 +207,163 @@ export class GarnetClient { return this.getClient().xAdd(key, "*", fields, trimOptions); } + async createConsumerGroup( + key: string, + group: string, + startId = "$", + ): Promise { + await this.getClient().xGroupCreate(key, group, startId, { + MKSTREAM: true, + }); + } + + async readGroup( + key: string, + group: string, + consumer: string, + startId: string, + options?: { + blockMs?: number | undefined; + count?: number | undefined; + }, + ): Promise { + const readOptions: { + BLOCK?: number; + COUNT?: number; + } = {}; + if (options?.blockMs !== undefined) { + readOptions.BLOCK = options.blockMs; + } + if (options?.count !== undefined) { + readOptions.COUNT = options.count; + } + + const reply = await this.getClient().xReadGroup( + group, + consumer, + { + key, + id: startId, + }, + readOptions, + ); + + if (!reply) { + return null; + } + + return (reply as RawStreamReadResult[]).map((stream) => ({ + name: String(stream.name), + messages: stream.messages.map((message) => ({ + deliveriesCounter: + typeof message.deliveriesCounter === "number" + ? message.deliveriesCounter + : undefined, + id: String(message.id), + message: normalizeStreamFields(message.message), + millisElapsedFromDelivery: + typeof message.millisElapsedFromDelivery === "number" + ? message.millisElapsedFromDelivery + : undefined, + })), + })); + } + + async acknowledgeStream( + key: string, + group: string, + ids: string | string[], + ): Promise { + return this.getClient().xAck(key, group, ids); + } + + async getPendingStreamRange( + key: string, + group: string, + count: number, + options?: { + consumer?: string | undefined; + idleMs?: number | undefined; + start?: string | undefined; + end?: string | undefined; + }, + ): Promise { + const pendingOptions: { + IDLE?: number; + consumer?: string; + } = {}; + if (options?.idleMs !== undefined) { + pendingOptions.IDLE = options.idleMs; + } + if (options?.consumer !== undefined) { + pendingOptions.consumer = options.consumer; + } + + const reply = await this.getClient().xPendingRange( + key, + group, + options?.start ?? "-", + options?.end ?? "+", + count, + pendingOptions, + ); + + return reply.map((entry) => ({ + consumer: String(entry.consumer), + deliveriesCounter: entry.deliveriesCounter, + id: String(entry.id), + millisecondsSinceLastDelivery: entry.millisecondsSinceLastDelivery, + })); + } + + async claimPendingStream( + key: string, + group: string, + consumer: string, + minIdleTimeMs: number, + ids: string | string[], + options?: { + retryCount?: number | undefined; + }, + ): Promise { + const claimOptions: { + RETRYCOUNT?: number; + } = {}; + if (options?.retryCount !== undefined) { + claimOptions.RETRYCOUNT = options.retryCount; + } + + const claimed = await this.getClient().xClaim( + key, + group, + consumer, + minIdleTimeMs, + ids, + claimOptions, + ); + + return claimed.flatMap((message) => { + if (!message) { + return []; + } + + return [ + { + deliveriesCounter: + typeof message.deliveriesCounter === "number" + ? message.deliveriesCounter + : undefined, + id: String(message.id), + message: normalizeStreamFields(message.message), + millisElapsedFromDelivery: + typeof message.millisElapsedFromDelivery === "number" + ? message.millisElapsedFromDelivery + : undefined, + }, + ]; + }); + } + async scanKeys(pattern: string): Promise { const keys: string[] = []; diff --git a/src/common/nats-subjects.ts b/src/common/nats-subjects.ts new file mode 100644 index 0000000..2459847 --- /dev/null +++ b/src/common/nats-subjects.ts @@ -0,0 +1,19 @@ +import type { EvalReport } from "@eval/types"; + +export const AGENT_REQUEST_SUBJECT = "lolympus-agent.request"; +export const AGENT_ALERT_SUBJECT = "lolympus-agent.alert"; +export const AGENT_EVAL_REPORT_SUBJECT = "lolympus-agent.eval.report"; + +export interface AlertPayload { + kind: "latency" | "eval-score-drop" | "llm-error-rate" | "endpoint-down"; + metric: string; + severity: "info" | "warn" | "critical"; + suggestedAction?: string | undefined; + threshold: number; + timestamp: string; + traceId?: string | undefined; + value: number; + window: string; +} + +export type EvalReportPayload = EvalReport; diff --git a/src/config.ts b/src/config.ts index 4dcd8aa..3fe9da2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -65,8 +65,12 @@ export function getOtelSamplingRatio(): number { return getRatioNumber("OTEL_SAMPLING_RATIO", fallback); } -export function getOtelExporter(): "stdout" { - return "stdout"; +export function getOtelExporter(): "stdout" | "none" { + const env = process.env["OTEL_EXPORTER"]?.trim().toLowerCase(); + if (env === "none" || env === "stdout") { + return env; + } + return getNodeEnv() === "dev" ? "none" : "stdout"; } export function getRequestSubject(): string { @@ -75,6 +79,10 @@ export function getRequestSubject(): string { ); } +export function getAgentRequestTimeoutMs(): number { + return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000); +} + export function getOnboardingSessionKeyPrefix(): string { return process.env["ONBOARDING_SESSION_KEY_PREFIX"]?.trim() || "lolympus"; } @@ -95,6 +103,10 @@ export function getBackgroundStreamMaxLen(): number { return getPositiveNumber("BACKGROUND_STREAM_MAXLEN", 5_000); } +export function getEvalStreamMaxLen(): number { + return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000); +} + export function getMaxConversationMessages(): number { return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20); } @@ -123,14 +135,110 @@ export function getClassifierMaxTokens(): number { return getPositiveNumber("CLASSIFIER_MAX_TOKENS", 16); } +export function getLlmJudgeModel(): string { + return process.env["LLM_JUDGE_MODEL"]?.trim() || "gpt-5.3-codex-spark"; +} + +export function getLlmJudgeMaxTokens(): number { + return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128); +} + +export function getSafetyMarginMs(): number { + return getPositiveNumber("SAFETY_MARGIN_MS", 500); +} + export function getDeadlineNearThresholdMin(): number { return getPositiveNumber("DEADLINE_NEAR_THRESHOLD_MIN", 30); } +export function getOpenRouterEnabled(): boolean { + const value = process.env["OPENROUTER_ENABLED"]?.trim().toLowerCase(); + return value === "1" || value === "true" || value === "yes"; +} + +export function getOpenRouterApiUrl(): string { + return process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1"; +} + +export function getTier0Model(): string { + return ( + process.env["TIER0_MODEL"]?.trim() || + process.env["CLI_PROXY_LLM_MODEL"]?.trim() || + "gpt-5.3-codex-spark" + ); +} + +export function getTier1PlannerModel(): string { + return ( + process.env["TIER1_PLANNER_MODEL"]?.trim() || + process.env["CLI_PROXY_LLM_MODEL"]?.trim() || + "gpt-5.4" + ); +} + +export function getTier1ResponderModel(): string { + return ( + process.env["TIER1_RESPONDER_MODEL"]?.trim() || + process.env["CLI_PROXY_LLM_MODEL"]?.trim() || + "gpt-5.3-codex-spark" + ); +} + +export function getTier2CriticModel(): string { + return ( + process.env["TIER2_CRITIC_MODEL"]?.trim() || + process.env["CLI_PROXY_LLM_MODEL"]?.trim() || + "gpt-5.4" + ); +} + +export function getOpenRouterTier1Model(): string { + return ( + process.env["OPENROUTER_TIER1_MODEL"]?.trim() || + "openai/gpt-5.3-codex-spark" + ); +} + +export function getOpenRouterTier2Model(): string { + return process.env["OPENROUTER_TIER2_MODEL"]?.trim() || "openai/gpt-5.4"; +} + export function getLlmMaxTokens(): number { return getPositiveNumber("LLM_MAX_TOKENS", 768); } +export function getTier0MaxTokens(): number { + return getPositiveNumber("TIER0_MAX_TOKENS", 384); +} + +export function getTier1MaxTokens(): number { + return getPositiveNumber("TIER1_MAX_TOKENS", 512); +} + +export function getTier2MaxTokens(): number { + return getPositiveNumber("TIER2_MAX_TOKENS", 512); +} + +export function getPlannerMaxTokens(): number { + return getPositiveNumber("PLANNER_MAX_TOKENS", 256); +} + +export function getCriticMaxTokens(): number { + return getPositiveNumber("CRITIC_MAX_TOKENS", 512); +} + +export function getTier0EstimatedCallMs(): number { + return getPositiveNumber("TIER0_ESTIMATED_CALL_MS", 600); +} + +export function getTier1EstimatedCallMs(): number { + return getPositiveNumber("TIER1_ESTIMATED_CALL_MS", 1200); +} + +export function getTier2EstimatedCallMs(): number { + return getPositiveNumber("TIER2_ESTIMATED_CALL_MS", 1400); +} + export function getLlmTemperature(): number { const raw = process.env["LLM_TEMPERATURE"]?.trim(); if (raw) { @@ -151,3 +259,39 @@ export function getMongoDbProjectId(): string | undefined { export function getRoomContextLimit(): number { return getPositiveNumber("ROOM_CONTEXT_LIMIT", 8); } + +export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number { + switch (tier) { + case 0: + return getRatioNumber( + "SHADOW_EVAL_SAMPLING_RATE_TIER0", + getNodeEnv() === "prod" ? 0.1 : 1.0, + ); + case 1: + return getRatioNumber( + "SHADOW_EVAL_SAMPLING_RATE_TIER1", + getNodeEnv() === "prod" ? 0.5 : 1.0, + ); + case 2: + return getRatioNumber( + "SHADOW_EVAL_SAMPLING_RATE_TIER2", + 1.0, + ); + } +} + +export function getAlertThresholdP95LatencyMs(): number { + return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000); +} + +export function getAlertThresholdEvalScoreDropPct(): number { + return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10); +} + +export function getAlertThresholdLlmErrorRate(): number { + return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1); +} + +export function getAlertRateLimitSec(): number { + return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300); +} diff --git a/src/conversation/conversation-store.ts b/src/conversation/conversation-store.ts index 287bea9..28afd32 100644 --- a/src/conversation/conversation-store.ts +++ b/src/conversation/conversation-store.ts @@ -1,5 +1,6 @@ import type { ConversationMessage } from "@agent/types"; import type { GarnetClient } from "@common/garnet"; +import { getRequestLogger } from "@obs/context"; import { getBackgroundStreamMaxLen, getConversationTtlSeconds, @@ -52,17 +53,24 @@ export class ConversationStore { await this.garnetClient.setJson(key, trimmed, getConversationTtlSeconds()); if (trimmed.length > 5) { - await this.garnetClient.appendStream( - BACKGROUND_STREAM_KEY, - { - type: "summarize", - chatroomId, - userId, - }, - { - maxLen: getBackgroundStreamMaxLen(), - }, - ); + void this.garnetClient + .appendStream( + BACKGROUND_STREAM_KEY, + { + type: "summarize", + chatroomId, + userId, + }, + { + maxLen: getBackgroundStreamMaxLen(), + }, + ) + .catch((error) => { + getRequestLogger().warn( + { err: error, chatroomId, userId }, + "[ConversationStore] Failed to publish summarize job", + ); + }); } } diff --git a/src/conversation/summarizer.ts b/src/conversation/summarizer.ts index 7eb8b62..70e9420 100644 --- a/src/conversation/summarizer.ts +++ b/src/conversation/summarizer.ts @@ -8,7 +8,7 @@ function formatHistory(history: ConversationMessage[]): string { export async function summarizeConversation( llmClient: LlmClient, history: ConversationMessage[], - existingSummary?: string | undefined, + existingSummary?: string, ): Promise { if (!llmClient.hasEndpoints() || history.length === 0) { return existingSummary ?? ""; diff --git a/src/eval/cli.ts b/src/eval/cli.ts index 8621417..7d1cca3 100644 --- a/src/eval/cli.ts +++ b/src/eval/cli.ts @@ -1,9 +1,3 @@ -function printPhaseNotice(command: string): void { - console.info( - `[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`, - ); -} - function parsePositiveInteger(value: string | undefined): number | undefined { if (!value?.trim()) { return undefined; @@ -13,26 +7,181 @@ function parsePositiveInteger(value: string | undefined): number | undefined { return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; } +interface ParsedArgs { + command: string; + flags: Map; + positionals: string[]; +} + +function parseArgs(argv: string[]): ParsedArgs { + const [command = "run", ...rest] = argv; + const flags = new Map(); + const positionals: string[] = []; + + for (let index = 0; index < rest.length; index++) { + const token = rest[index]; + if (!token) { + continue; + } + + if (!token.startsWith("--")) { + positionals.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const equalsIndex = withoutPrefix.indexOf("="); + if (equalsIndex >= 0) { + const key = withoutPrefix.slice(0, equalsIndex); + const value = withoutPrefix.slice(equalsIndex + 1); + flags.set(key, value || true); + continue; + } + + const next = rest[index + 1]; + if (next && !next.startsWith("--")) { + flags.set(withoutPrefix, next); + index += 1; + continue; + } + + flags.set(withoutPrefix, true); + } + + return { + command, + flags, + positionals, + }; +} + +function getFlagString( + flags: Map, + key: string, +): string | undefined { + const value = flags.get(key); + return typeof value === "string" ? value : undefined; +} + +function hasFlag( + flags: Map, + key: string, +): boolean { + return flags.has(key); +} + +function printRunSummary(report: { + generatedAt: string; + summary: { + cases: number; + passRate: number; + ruleAverageScore: number; + averageLatencyMs: number; + llmAverageOverall: number | null; + llmJudgedCases: number; + }; + baselineDelta?: { + passRate: number | null; + ruleAverageScore: number | null; + llmAverageOverall: number | null; + averageLatencyMs: number | null; + } | undefined; +}, reportPath: string): void { + const llmLine = + report.summary.llmAverageOverall === null + ? "n/a" + : `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`; + + console.info(`[eval] generatedAt=${report.generatedAt}`); + console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`); + console.info( + `[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`, + ); + console.info(`[eval] llmAverageOverall=${llmLine}`); + + if (report.baselineDelta) { + console.info( + `[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`, + ); + } + + console.info(`[eval] report=${reportPath}`); +} + async function main(): Promise { const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader"); loadEnvironmentFromNodeEnv(); const { getLogger, initLogger } = await import("@obs/logger"); initLogger(); - const { captureGoldenSetScaffold } = await import("./capture.js"); + const logger = getLogger(); - const [command = "run", arg1, arg2] = process.argv.slice(2); + const parsed = parseArgs(process.argv.slice(2)); + const { command, flags, positionals } = parsed; - if (command === "capture") { - const outputPath = await captureGoldenSetScaffold({ - limit: parsePositiveInteger(arg1), - outputPath: arg2, - }); - getLogger().info({ outputPath }, '[eval] "capture" completed'); - return; + try { + if (command === "capture") { + const { captureGoldenSetScaffold } = await import("./capture.js"); + const outputPath = await captureGoldenSetScaffold({ + limit: + parsePositiveInteger(getFlagString(flags, "limit")) ?? + parsePositiveInteger(positionals[0]), + outputPath: getFlagString(flags, "output") ?? positionals[1], + }); + logger.info({ outputPath }, '[eval] "capture" completed'); + return; + } + + if (command === "run") { + const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import( + "./runner.js" + ); + const { publishEvalReportIfPossible } = await import( + "./runner.js" + ); + const setName = getFlagString(flags, "set") ?? "golden"; + const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName); + const limit = parsePositiveInteger(getFlagString(flags, "limit")); + + if (hasFlag(flags, "dry-run")) { + const cases = await loadEvalCases(inputPath, { limit }); + console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`); + return; + } + + const { report, reportPath } = await runOnboardingEval({ + enableLlmJudge: !hasFlag(flags, "skip-llm-judge"), + inputPath, + limit, + outputPath: getFlagString(flags, "output"), + setName, + }); + const publishedReport = hasFlag(flags, "no-publish-report") + ? false + : await publishEvalReportIfPossible(report); + printRunSummary(report, reportPath); + console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`); + return; + } + + if (command === "baseline") { + const { updateEvalBaseline } = await import("./runner.js"); + const reportPath = getFlagString(flags, "report") ?? positionals[0]; + const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath); + console.info( + `[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`, + ); + return; + } + + throw new Error(`Unknown eval command: ${command}`); + } catch (error) { + logger.error( + { err: error }, + `[eval] "${command}" failed`, + ); + process.exitCode = 1; } - - printPhaseNotice(command); } void main(); diff --git a/src/eval/golden/onboarding.jsonl b/src/eval/golden/onboarding.jsonl new file mode 100644 index 0000000..8faa5b5 --- /dev/null +++ b/src/eval/golden/onboarding.jsonl @@ -0,0 +1,3 @@ +{"id":"greeting-short","name":"Greeting should stay short and welcoming","input":{"request":{"requestId":"eval_greeting_1","skill":"onboarding","chatroomId":"room_eval_1","userId":"user_eval_1","nickname":"","trigger":"user-message","userMessage":"안녕하세요"},"session":null,"conversationHistory":[]},"expected":{"intent":"greeting","responseMode":"llm","mustIncludeAny":["환영","반갑"],"mustNotInclude":["디스코드"]}} +{"id":"status-photo-remaining","name":"Status question should answer only the remaining step","input":{"request":{"requestId":"eval_status_1","skill":"onboarding","chatroomId":"room_eval_2","userId":"user_eval_2","nickname":"프리리/95/여/D","trigger":"user-message","userMessage":"이제 뭐 남았지"},"session":{"chatroomId":"room_eval_2","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/D","nicknameVerified":true,"photoVerified":false,"status":"pending","userId":"user_eval_2"},"conversationHistory":[{"role":"user","content":"안녕하세요","timestamp":"2026-04-16T00:00:00.000Z"},{"role":"assistant","content":"환영합니다! 반갑습니다.","timestamp":"2026-04-16T00:00:01.000Z"}]},"expected":{"intent":"status-question","responseMode":"llm","mustIncludeAny":["남았","하트"],"mustNotInclude":["디스코드"]}} +{"id":"nickname-invalid-tier","name":"Nickname invalid flow should focus on nickname correction","input":{"request":{"requestId":"eval_nick_1","skill":"onboarding","chatroomId":"room_eval_3","userId":"user_eval_3","nickname":"프리리/95/여/다이아","trigger":"nickname-invalid"},"session":{"chatroomId":"room_eval_3","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/다이아","nicknameVerified":false,"photoVerified":false,"status":"pending","userId":"user_eval_3"},"conversationHistory":[]},"expected":{"intent":"nickname-question","responseMode":"llm-with-tools","mustIncludeAny":["닉네임","티어","형식"],"mustNotInclude":["디스코드"]}} diff --git a/src/eval/judges/llm-judge.test.ts b/src/eval/judges/llm-judge.test.ts new file mode 100644 index 0000000..193ae93 --- /dev/null +++ b/src/eval/judges/llm-judge.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; + +import { LlmClient } from "@llm/llm-client"; +import type { + ChatCompletionOptions, + ChatCompletionResult, + ChatMessage, +} from "@llm/types"; +import { judgeOnboardingWithLlm } from "./llm-judge.js"; +import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js"; + +class FakeLlmClient extends LlmClient { + constructor(private readonly payload: string | null) { + super({ endpoints: [] }); + } + + hasEndpoints(): boolean { + return this.payload !== null; + } + + chatCompletion( + _messages: ChatMessage[], + _options?: ChatCompletionOptions, + ): Promise { + if (this.payload === null) { + return Promise.resolve(null); + } + + return Promise.resolve({ + content: this.payload, + endpoint: "http://judge.test/v1/chat/completions", + message: { + role: "assistant", + content: this.payload, + }, + provider: "cliproxy", + raw: {}, + }); + } +} + +const evalCase: EvalCaseDefinition = { + id: "case-1", + lineNumber: 1, + input: { + request: { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "프리리/95/여/D", + trigger: "user-message", + userMessage: "이제 뭐 남았지", + }, + }, + expected: { + intent: "status-question", + responseMode: "llm", + }, +}; + +const derived: EvalDerivedSnapshot = { + allowShortResponse: true, + intent: "status-question", + phase: "photo-pending", + responseMode: "llm", + shouldRespond: true, + suggestedMaxTokens: 128, + tier: 0, +}; + +describe("judgeOnboardingWithLlm", () => { + it("parses a valid JSON judge response", async () => { + const result = await judgeOnboardingWithLlm( + new FakeLlmClient( + JSON.stringify({ + friendliness: 5, + accuracy: 4, + concision: 5, + scope: 4, + rationale: "질문 범위에 맞게 짧고 정확하게 답했다.", + }), + ), + evalCase, + "이제 방 하트 인증만 남았어요.", + derived, + ); + + expect(result.skipped).toBe(false); + if (!result.skipped) { + expect(result.scores.overall).toBe(4.5); + expect(result.provider).toBe("cliproxy"); + } + }); + + it("skips when the judge returns invalid JSON", async () => { + const result = await judgeOnboardingWithLlm( + new FakeLlmClient('{"friendliness":"bad"}'), + evalCase, + "이제 방 하트 인증만 남았어요.", + derived, + ); + + expect(result.skipped).toBe(true); + if (result.skipped) { + expect(result.reason).toBe("invalid-judge-json"); + } + }); +}); diff --git a/src/eval/judges/llm-judge.ts b/src/eval/judges/llm-judge.ts new file mode 100644 index 0000000..9c70c71 --- /dev/null +++ b/src/eval/judges/llm-judge.ts @@ -0,0 +1,134 @@ +import { z } from "zod"; +import type { LlmClient } from "@llm/llm-client"; +import { getLlmJudgeMaxTokens, getLlmJudgeModel } from "../../config.js"; +import type { + EvalCaseDefinition, + EvalDerivedSnapshot, + LlmJudgeResult, + LlmJudgeScores, +} from "../types.js"; + +const llmJudgeSchema = z.object({ + friendliness: z.number().int().min(1).max(5), + accuracy: z.number().int().min(1).max(5), + concision: z.number().int().min(1).max(5), + scope: z.number().int().min(1).max(5), + rationale: z.string().trim().min(1).max(200), +}); + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +function buildJudgePrompt( + evalCase: EvalCaseDefinition, + responseText: string, + derived: EvalDerivedSnapshot, +): string { + return [ + "다음 온보딩 에이전트 응답을 평가해 주세요.", + "평가 축은 friendliness, accuracy, concision, scope 입니다.", + "각 점수는 1~5의 정수만 사용하세요.", + "scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.", + "", + "[request]", + JSON.stringify(evalCase.input.request), + "", + "[expected]", + JSON.stringify({ + intent: evalCase.expected.intent ?? null, + mustIncludeAny: evalCase.expected.mustIncludeAny ?? [], + mustNotInclude: evalCase.expected.mustNotInclude ?? [], + responseMode: evalCase.expected.responseMode ?? null, + }), + "", + "[derived]", + JSON.stringify(derived), + "", + "[assistant_response]", + responseText, + "", + 'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}', + ].join("\n"); +} + +export async function judgeOnboardingWithLlm( + llmClient: LlmClient, + evalCase: EvalCaseDefinition, + responseText: string, + derived: EvalDerivedSnapshot, +): Promise { + if (!responseText.trim()) { + return { + skipped: true, + reason: "empty-response", + }; + } + + if (!llmClient.hasEndpoints()) { + return { + skipped: true, + reason: "no-llm-endpoint", + }; + } + + const response = await llmClient.chatCompletion( + [ + { + role: "system", + content: + "당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.", + }, + { + role: "user", + content: buildJudgePrompt(evalCase, responseText, derived), + }, + ], + { + jsonMode: true, + maxTokens: getLlmJudgeMaxTokens(), + modelOverride: getLlmJudgeModel(), + temperature: 0, + }, + ); + + if (!response) { + return { + skipped: true, + reason: "judge-call-failed", + }; + } + + try { + const parsed = llmJudgeSchema.parse(JSON.parse(response.content)); + const scores: LlmJudgeScores = { + friendliness: parsed.friendliness, + accuracy: parsed.accuracy, + concision: parsed.concision, + scope: parsed.scope, + overall: round( + (parsed.friendliness + + parsed.accuracy + + parsed.concision + + parsed.scope) / + 4, + ), + }; + + return { + skipped: false, + model: getLlmJudgeModel(), + endpoint: response.endpoint, + provider: response.provider, + scores, + rationale: parsed.rationale, + raw: response.content, + }; + } catch { + return { + skipped: true, + reason: "invalid-judge-json", + raw: response.content, + }; + } +} diff --git a/src/eval/judges/rule-judge.test.ts b/src/eval/judges/rule-judge.test.ts new file mode 100644 index 0000000..677133e --- /dev/null +++ b/src/eval/judges/rule-judge.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { judgeOnboardingRuleCase } from "./rule-judge.js"; +import type { DerivedState } from "@skills/onboarding/state/onboarding-state"; + +function makeDerived( + overrides?: Partial, +): DerivedState { + return { + phase: { kind: "photo-pending" }, + intent: { kind: "status-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, + ...overrides, + }; +} + +describe("judgeOnboardingRuleCase", () => { + it("passes when active checks match", () => { + const result = judgeOnboardingRuleCase( + { + intent: "status-question", + responseMode: "llm", + mustIncludeAny: ["방 하트 인증"], + mustNotInclude: ["디스코드 서버"], + }, + "이제 방 하트 인증만 남았어요.", + makeDerived(), + ); + + expect(result.passed).toBe(true); + expect(result.score).toBe(1); + expect(result.validatorStatus).toBe("pass"); + }); + + it("fails intent and mustNotInclude mismatches", () => { + const result = judgeOnboardingRuleCase( + { + intent: "help-how", + mustNotInclude: ["디스코드"], + }, + "디스코드 서버에 먼저 들어오시면 돼요.", + makeDerived(), + ); + + expect(result.passed).toBe(false); + expect(result.checks.find((check) => check.name === "intent")?.passed).toBe( + false, + ); + expect( + result.checks.find((check) => check.name === "mustNotInclude")?.passed, + ).toBe(false); + }); + + it("treats rejecting validator results as a failed rule check", () => { + const result = judgeOnboardingRuleCase( + {}, + "디스코드 서버에 들어가시면 돼요.", + makeDerived({ + intent: { kind: "photo-question" }, + }), + ); + + expect(result.passed).toBe(false); + expect(result.validatorStatus).toBe("reject"); + }); +}); diff --git a/src/eval/judges/rule-judge.ts b/src/eval/judges/rule-judge.ts new file mode 100644 index 0000000..0baecfe --- /dev/null +++ b/src/eval/judges/rule-judge.ts @@ -0,0 +1,121 @@ +import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator"; +import type { DerivedState } from "@skills/onboarding/state/onboarding-state"; +import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js"; + +function normalizeText(value: string): string { + return value.trim().toLowerCase(); +} + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +export function judgeOnboardingRuleCase( + expected: EvalCaseExpected, + responseText: string, + derived: DerivedState, +): RuleJudgeResult { + const validation = validateOnboardingResponse(responseText, derived, { + allowShortResponse: derived.allowShortResponse, + }); + const checks: RuleJudgeCheck[] = []; + + checks.push({ + name: "validator", + passed: validation.status !== "reject", + skipped: false, + detail: + validation.reasons.length > 0 + ? validation.reasons.join(", ") + : validation.status, + }); + + const expectedIntent = expected.intent?.trim(); + if (expectedIntent) { + checks.push({ + name: "intent", + passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind), + skipped: false, + detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`, + }); + } else { + checks.push({ + name: "intent", + passed: true, + skipped: true, + }); + } + + const expectedResponseMode = expected.responseMode?.trim(); + if (expectedResponseMode) { + checks.push({ + name: "responseMode", + passed: + normalizeText(expectedResponseMode) === + normalizeText(derived.responseMode), + skipped: false, + detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`, + }); + } else { + checks.push({ + name: "responseMode", + passed: true, + skipped: true, + }); + } + + const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? []; + if (mustIncludeAny.length > 0) { + const normalizedResponse = normalizeText(responseText); + const matched = mustIncludeAny.find((candidate) => + normalizedResponse.includes(normalizeText(candidate)), + ); + + checks.push({ + name: "mustIncludeAny", + passed: Boolean(matched), + skipped: false, + detail: matched + ? `matched=${matched}` + : `expected one of: ${mustIncludeAny.join(", ")}`, + }); + } else { + checks.push({ + name: "mustIncludeAny", + passed: true, + skipped: true, + }); + } + + const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? []; + if (mustNotInclude.length > 0) { + const normalizedResponse = normalizeText(responseText); + const matched = mustNotInclude.find((candidate) => + normalizedResponse.includes(normalizeText(candidate)), + ); + + checks.push({ + name: "mustNotInclude", + passed: !matched, + skipped: false, + detail: matched ? `forbidden=${matched}` : undefined, + }); + } else { + checks.push({ + name: "mustNotInclude", + passed: true, + skipped: true, + }); + } + + const activeChecks = checks.filter((check) => !check.skipped); + const passedChecks = activeChecks.filter((check) => check.passed).length; + + return { + passed: activeChecks.every((check) => check.passed), + score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length), + checks, + validatorStatus: validation.status, + validatorReasons: validation.reasons, + }; +} diff --git a/src/eval/runner.test.ts b/src/eval/runner.test.ts new file mode 100644 index 0000000..28cc16f --- /dev/null +++ b/src/eval/runner.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it, vi } from "vitest"; + +import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects"; +import { publishEvalReportIfPossible } from "./runner.js"; +import type { EvalReport } from "./types.js"; + +const natsMocks = vi.hoisted(() => ({ + close: vi.fn().mockResolvedValue(undefined), + connect: vi.fn().mockResolvedValue(undefined), + publish: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@common/nats", () => { + return { + NatsClient: class { + connect = natsMocks.connect; + publish = natsMocks.publish; + close = natsMocks.close; + }, + }; +}); + +const report: EvalReport = { + generatedAt: "2026-04-17T00:00:00.000Z", + inputPath: "/tmp/onboarding.jsonl", + setName: "golden", + summary: { + averageLatencyMs: 100, + cases: 1, + emptyResponses: 0, + llmAverageAccuracy: 4, + llmAverageConcision: 4, + llmAverageFriendliness: 5, + llmAverageOverall: 4.25, + llmAverageScope: 4, + llmJudgedCases: 1, + passRate: 1, + passedCases: 1, + ruleAverageScore: 1, + }, + cases: [], +}; + +describe("publishEvalReportIfPossible", () => { + it("publishes the report to the eval report subject", async () => { + const result = await publishEvalReportIfPossible(report); + + expect(result).toBe(true); + expect(natsMocks.connect).toHaveBeenCalledTimes(1); + expect(natsMocks.publish).toHaveBeenCalledWith( + AGENT_EVAL_REPORT_SUBJECT, + report, + ); + expect(natsMocks.close).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/eval/runner.ts b/src/eval/runner.ts new file mode 100644 index 0000000..d13a3f0 --- /dev/null +++ b/src/eval/runner.ts @@ -0,0 +1,842 @@ +import { existsSync } from "node:fs"; +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { basename, dirname, resolve } from "node:path"; + +import type { AgentRequest } from "@agent/types"; +import { GarnetClient } from "@common/garnet"; +import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects"; +import { NatsClient } from "@common/nats"; +import { getConversationSummaryKey } from "@conv/conversation-store"; +import { LlmClient } from "@llm/llm-client"; +import type { ProviderName } from "@llm/types"; +import { runWithContext } from "@obs/context"; +import { getLogger } from "@obs/logger"; +import type { RoomMessage } from "@room/types"; +import { RoomContextService } from "@room/room-context-service"; +import { OnboardingSkill } from "@skills/onboarding/index"; +import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; +import { OnboardingIntentClassifier } from "@skills/onboarding/state/classifier"; +import { deriveState } from "@skills/onboarding/state/derive"; +import type { + ClassifiedIntent, + DerivedState, +} from "@skills/onboarding/state/onboarding-state"; +import { noopShadowEvalPublisher } from "./shadow-bus.js"; +import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl, getOpenRouterEnabled } from "../config.js"; +import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; +import { judgeOnboardingRuleCase } from "./judges/rule-judge.js"; +import type { + EvalBaselineFile, + EvalCaseDefinition, + EvalCaseExpected, + EvalCaseInput, + EvalCaseResult, + EvalClassifiedSnapshot, + EvalDerivedSnapshot, + EvalReport, + EvalReportSummary, + EvalSummaryDelta, +} from "./types.js"; + +const DEFAULT_REPORTS_DIR = "eval-reports"; +const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`; +const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl"; +const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl"; +const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline"; + +class InMemoryGarnetClient extends GarnetClient { + private readonly hashStore = new Map>(); + private readonly stringStore = new Map(); + + getJson(key: string): Promise { + const raw = this.stringStore.get(key); + return Promise.resolve(raw ? (JSON.parse(raw) as T) : null); + } + + setJson( + key: string, + value: unknown, + _ttlSeconds?: number, + ): Promise { + this.stringStore.set(key, JSON.stringify(value)); + return Promise.resolve(); + } + + getString(key: string): Promise { + return Promise.resolve(this.stringStore.get(key) ?? null); + } + + setString( + key: string, + value: string, + _ttlSeconds?: number, + ): Promise { + this.stringStore.set(key, value); + return Promise.resolve(); + } + + delete(key: string): Promise { + const stringDeleted = this.stringStore.delete(key); + const hashDeleted = this.hashStore.delete(key); + return Promise.resolve(stringDeleted || hashDeleted ? 1 : 0); + } + + getHash(key: string): Promise> { + return Promise.resolve({ ...(this.hashStore.get(key) ?? {}) }); + } + + setHash( + key: string, + fields: Record, + ): Promise { + this.hashStore.set(key, { ...fields }); + return Promise.resolve(); + } +} + +class InMemoryRoomContextService extends RoomContextService { + constructor(private readonly messages: RoomMessage[]) { + super(); + } + + getRecentMessages( + _chatroomId: string, + limit?: number, + ): Promise { + const effectiveLimit = limit ?? this.messages.length; + return Promise.resolve(this.messages.slice(-effectiveLimit)); + } +} + +interface RawEvalCaseLine { + expected?: Record; + id?: unknown; + input?: Record; + name?: unknown; + notes?: unknown; +} + +export interface EvalRunOptions { + enableLlmJudge?: boolean | undefined; + inputPath?: string | undefined; + limit?: number | undefined; + outputPath?: string | undefined; + setName?: string | undefined; +} + +function round(value: number): number { + return Number(value.toFixed(3)); +} + +function average(values: number[]): number | null { + if (values.length === 0) { + return null; + } + return round(values.reduce((sum, value) => sum + value, 0) / values.length); +} + +function averageOrZero(values: number[]): number { + return values.length === 0 ? 0 : round(values.reduce((sum, value) => sum + value, 0) / values.length); +} + +function firstNonEmpty( + ...values: Array +): string | undefined { + for (const value of values) { + if (value?.trim()) { + return value.trim(); + } + } + return undefined; +} + +function parseEndpointList( + ...values: Array +): string[] { + const endpoints: string[] = []; + + for (const value of values) { + if (!value?.trim()) { + continue; + } + + for (const item of value.split(/[\n,]/u)) { + const trimmed = item.trim(); + if (trimmed) { + endpoints.push(trimmed); + } + } + } + + return [...new Set(endpoints)]; +} + +function toProviderEndpoints( + urls: string[], + provider?: ProviderName, +): Array { + if (!provider) { + return urls; + } + + return urls.map((url) => ({ url, provider })); +} + +function createEvalLlmClient(): LlmClient { + const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]); + const llamaEndpoints = parseEndpointList( + process.env["AGENT_LLM_API_URLS"], + process.env["LLAMA_SERVER_API_URL"], + ); + const openRouterEndpoints = getOpenRouterEnabled() + ? parseEndpointList(process.env["OPENROUTER_API_URL"], getOpenRouterApiUrl()) + : []; + + const llmApiKey = firstNonEmpty( + process.env["AGENT_LLM_API_KEY"], + process.env["LLAMA_SERVER_API_KEY"], + ); + const providerApiKeys = Object.fromEntries( + [ + ["cliproxy", firstNonEmpty(process.env["CLI_PROXY_API_KEY"])], + ["llama", llmApiKey], + ["openrouter", firstNonEmpty(process.env["OPENROUTER_API_KEY"])], + ].filter((entry): entry is [string, string] => typeof entry[1] === "string"), + ) as Partial>; + + return new LlmClient({ + endpoints: [ + ...toProviderEndpoints(cliproxyEndpoints, "cliproxy"), + ...toProviderEndpoints(openRouterEndpoints, "openrouter"), + ...toProviderEndpoints(llamaEndpoints, "llama"), + ], + apiKey: llmApiKey, + model: firstNonEmpty(process.env["AGENT_LLM_MODEL"]), + providerApiKeys, + }); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function getStringField(record: Record, key: string): string | undefined { + const value = record[key]; + return typeof value === "string" ? value : undefined; +} + +function getStringArrayField( + record: Record, + key: string, +): string[] | undefined { + const value = record[key]; + if (!Array.isArray(value)) { + return undefined; + } + + const strings = value.filter((item): item is string => typeof item === "string"); + return strings.length > 0 ? strings : []; +} + +function toAgentRequest(raw: Record): AgentRequest { + const requestId = getStringField(raw, "requestId"); + const skill = getStringField(raw, "skill"); + const chatroomId = getStringField(raw, "chatroomId"); + const userId = getStringField(raw, "userId"); + const nickname = getStringField(raw, "nickname"); + const trigger = getStringField(raw, "trigger"); + + if (!requestId || !skill || !chatroomId || !userId || !nickname || !trigger) { + throw new Error("input.request is missing required AgentRequest fields"); + } + + const request: AgentRequest = { + requestId, + skill, + chatroomId, + userId, + nickname, + trigger, + }; + + const userMessage = getStringField(raw, "userMessage"); + if (userMessage !== undefined) { + request.userMessage = userMessage; + } + + const context = raw["context"]; + if (isRecord(context)) { + request.context = context; + } + + return request; +} + +function parseConversationHistory(raw: unknown): EvalCaseInput["conversationHistory"] { + if (!Array.isArray(raw)) { + return undefined; + } + + return raw + .filter((item): item is Record => isRecord(item)) + .map((item) => { + const role = item["role"]; + const content = getStringField(item, "content") ?? ""; + const timestamp = getStringField(item, "timestamp") ?? new Date(0).toISOString(); + + if (role !== "user" && role !== "assistant") { + throw new Error("conversationHistory contains an invalid role"); + } + + return { + role, + content, + timestamp, + }; + }); +} + +function parseRoomContext(raw: unknown): EvalCaseInput["roomContext"] { + if (!Array.isArray(raw)) { + return undefined; + } + + return raw + .filter((item): item is Record => isRecord(item)) + .map((item) => ({ + senderId: getStringField(item, "senderId") ?? "", + nickname: getStringField(item, "nickname") ?? "알 수 없음", + message: getStringField(item, "message") ?? "", + timestamp: getStringField(item, "timestamp") ?? new Date(0).toISOString(), + })); +} + +function parseExpected(raw: Record | undefined): EvalCaseExpected { + if (!raw) { + return {}; + } + + const expected: EvalCaseExpected = {}; + const intent = getStringField(raw, "intent"); + const responseMode = getStringField(raw, "responseMode"); + const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny"); + const mustNotInclude = getStringArrayField(raw, "mustNotInclude"); + + if (intent) { + expected.intent = intent; + } + if (responseMode) { + expected.responseMode = responseMode; + } + if (mustIncludeAny) { + expected.mustIncludeAny = mustIncludeAny; + } + if (mustNotInclude) { + expected.mustNotInclude = mustNotInclude; + } + + return expected; +} + +function parseEvalCaseLine( + raw: RawEvalCaseLine, + lineNumber: number, + sourcePath: string, +): EvalCaseDefinition { + if (!isRecord(raw.input)) { + throw new Error("input is required"); + } + + const requestRaw = raw.input["request"]; + if (!isRecord(requestRaw)) { + throw new Error("input.request is required"); + } + + const input: EvalCaseInput = { + request: toAgentRequest(requestRaw), + }; + + const conversationHistory = parseConversationHistory(raw.input["conversationHistory"]); + if (conversationHistory) { + input.conversationHistory = conversationHistory; + } + + const roomContext = parseRoomContext(raw.input["roomContext"]); + if (roomContext) { + input.roomContext = roomContext; + } + + if (raw.input["summary"] !== undefined) { + const summary = getStringField(raw.input, "summary"); + if (summary !== undefined) { + input.summary = summary; + } + } + + if (raw.input["session"] === null || isRecord(raw.input["session"])) { + input.session = (raw.input["session"] ?? null) as OnboardingSession | null; + } + + if (raw.input["blackboard"] === null || isRecord(raw.input["blackboard"])) { + input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null; + } + + return { + id: + typeof raw.id === "string" && raw.id.trim() + ? raw.id.trim() + : `${basename(sourcePath)}:${lineNumber}`, + lineNumber, + name: typeof raw.name === "string" ? raw.name : undefined, + notes: typeof raw.notes === "string" ? raw.notes : undefined, + input, + expected: parseExpected(isRecord(raw.expected) ? raw.expected : undefined), + }; +} + +function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot { + return { + allowShortResponse: derived.allowShortResponse, + intent: derived.intent.kind, + phase: derived.phase.kind, + responseMode: derived.responseMode, + shouldRespond: derived.shouldRespond, + suggestedMaxTokens: derived.suggestedMaxTokens, + tier: derived.tier, + }; +} + +function toClassifiedSnapshot(classified: ClassifiedIntent): EvalClassifiedSnapshot { + return { + confidence: round(classified.confidence), + intent: classified.intent.kind, + source: classified.source, + }; +} + +function toRoomMessages( + input: EvalCaseInput["roomContext"], +): RoomMessage[] { + return ( + input?.map((message) => ({ + senderId: message.senderId, + nickname: message.nickname, + message: message.message, + timestamp: new Date(message.timestamp), + })) ?? [] + ); +} + +async function hydrateEvalState( + garnetClient: InMemoryGarnetClient, + input: EvalCaseInput, +): Promise { + const { + request: { chatroomId, userId }, + } = input; + + if (input.session !== undefined) { + await garnetClient.setJson( + `${getOnboardingSessionKeyPrefix()}:onboarding:${chatroomId}:${userId}`, + input.session, + ); + } + + if (input.blackboard !== undefined) { + await garnetClient.setJson( + `lolympus-agent:bb:${chatroomId}:${userId}`, + input.blackboard, + ); + } + + if (input.summary?.trim()) { + await garnetClient.setString( + getConversationSummaryKey(chatroomId, userId), + input.summary, + ); + } +} + +function createSummary(results: EvalCaseResult[]): EvalReportSummary { + const llmResults = results.flatMap((result) => + result.llmJudge.skipped ? [] : [result.llmJudge], + ); + + return { + averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)), + cases: results.length, + emptyResponses: results.filter((result) => !result.response.text.trim()).length, + llmAverageAccuracy: average( + llmResults.map((result) => result.scores.accuracy), + ), + llmAverageConcision: average( + llmResults.map((result) => result.scores.concision), + ), + llmAverageFriendliness: average( + llmResults.map((result) => result.scores.friendliness), + ), + llmAverageOverall: average( + llmResults.map((result) => result.scores.overall), + ), + llmAverageScope: average( + llmResults.map((result) => result.scores.scope), + ), + llmJudgedCases: llmResults.length, + passRate: + results.length === 0 + ? 0 + : round( + results.filter((result) => result.ruleJudge.passed).length / + results.length, + ), + passedCases: results.filter((result) => result.ruleJudge.passed).length, + ruleAverageScore: averageOrZero( + results.map((result) => result.ruleJudge.score), + ), + }; +} + +function createBaselineDelta( + current: EvalReportSummary, + baseline: EvalReportSummary, +): EvalSummaryDelta { + const subtract = ( + left: number | null, + right: number | null, + ): number | null => { + if (left === null || right === null) { + return null; + } + return round(left - right); + }; + + return { + averageLatencyMs: subtract(current.averageLatencyMs, baseline.averageLatencyMs), + llmAverageOverall: subtract( + current.llmAverageOverall, + baseline.llmAverageOverall, + ), + passRate: subtract(current.passRate, baseline.passRate), + ruleAverageScore: subtract(current.ruleAverageScore, baseline.ruleAverageScore), + }; +} + +function buildDefaultReportPath(timestamp: string): string { + const filename = `${timestamp.replace(/[:.]/gu, "-")}.json`; + return resolve(process.cwd(), DEFAULT_REPORTS_DIR, filename); +} + +async function writeReport( + report: EvalReport, + outputPath: string, +): Promise { + const resolved = resolve(process.cwd(), outputPath); + await mkdir(dirname(resolved), { recursive: true }); + await writeFile(resolved, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + return resolved; +} + +async function loadBaselineFile(): Promise { + const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE); + if (!existsSync(baselinePath)) { + return null; + } + + const raw = await readFile(baselinePath, "utf8"); + return JSON.parse(raw) as EvalBaselineFile; +} + +export function resolveEvalSetPath(setNameOrPath = "golden"): string { + if (setNameOrPath === "golden") { + return resolve(process.cwd(), GOLDEN_SET_PATH); + } + + if (setNameOrPath === "capture") { + return resolve(process.cwd(), CAPTURE_SET_PATH); + } + + if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) { + return resolve(process.cwd(), setNameOrPath); + } + + return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`); +} + +export async function loadEvalCases( + inputPath: string, + options?: { limit?: number | undefined }, +): Promise { + const resolvedPath = resolve(process.cwd(), inputPath); + if (!existsSync(resolvedPath)) { + throw new Error(`Eval set not found: ${resolvedPath}`); + } + + const raw = await readFile(resolvedPath, "utf8"); + const lines = raw.split(/\r?\n/u).filter((line) => line.trim()); + const cases = lines.map((line, index) => + parseEvalCaseLine( + JSON.parse(line) as RawEvalCaseLine, + index + 1, + resolvedPath, + ), + ); + + return options?.limit ? cases.slice(0, options.limit) : cases; +} + +async function evaluateCase( + evalCase: EvalCaseDefinition, + llmClient: LlmClient, + enableLlmJudge: boolean, +): Promise { + const input = evalCase.input; + const inMemoryGarnet = new InMemoryGarnetClient(); + await hydrateEvalState(inMemoryGarnet, input); + + const roomContextService = + input.roomContext && input.roomContext.length > 0 + ? new InMemoryRoomContextService(toRoomMessages(input.roomContext)) + : undefined; + const skill = new OnboardingSkill( + inMemoryGarnet, + llmClient, + roomContextService, + noopShadowEvalPublisher, + ); + const classifier = new OnboardingIntentClassifier(llmClient); + + return runWithContext( + { + chatroomId: input.request.chatroomId, + deadlineAt: new Date(Date.now() + 4_500).toISOString(), + requestId: input.request.requestId, + skill: input.request.skill, + startedAt: new Date().toISOString(), + trigger: input.request.trigger, + userId: input.request.userId, + }, + async () => { + const classified = await classifier.classify( + input.request.userMessage, + input.request.trigger, + ); + const derived = deriveState( + input.session ?? null, + input.request.trigger, + classified.intent, + ); + + const startedAt = Date.now(); + try { + const response = await skill.handle( + input.request, + input.conversationHistory ?? [], + ); + const latencyMs = Date.now() - startedAt; + const derivedSnapshot = toDerivedSnapshot(derived); + const ruleJudge = judgeOnboardingRuleCase( + evalCase.expected, + response.text, + derived, + ); + const llmJudge = enableLlmJudge + ? await judgeOnboardingWithLlm( + llmClient, + evalCase, + response.text, + derivedSnapshot, + ) + : { + skipped: true as const, + reason: "disabled", + }; + + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: toClassifiedSnapshot(classified), + derived: derivedSnapshot, + request: input.request, + response, + ruleJudge, + llmJudge, + }; + } catch (error) { + const latencyMs = Date.now() - startedAt; + const errorMessage = + error instanceof Error ? error.message : String(error); + + return { + id: evalCase.id, + lineNumber: evalCase.lineNumber, + name: evalCase.name, + latencyMs, + classified: toClassifiedSnapshot(classified), + derived: toDerivedSnapshot(derived), + request: input.request, + response: { + requestId: input.request.requestId, + text: "", + confidence: 0, + source: "template", + metadata: { + error: errorMessage, + }, + }, + ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived), + llmJudge: { + skipped: true, + reason: "skill-error", + }, + error: errorMessage, + }; + } + }, + ); +} + +export async function runOnboardingEval( + options?: EvalRunOptions, +): Promise<{ report: EvalReport; reportPath: string }> { + const logger = getLogger(); + const setName = options?.setName ?? "golden"; + const inputPath = options?.inputPath + ? resolve(process.cwd(), options.inputPath) + : resolveEvalSetPath(setName); + const evalCases = await loadEvalCases(inputPath, { + limit: options?.limit, + }); + + const llmClient = createEvalLlmClient(); + const enableLlmJudge = options?.enableLlmJudge ?? true; + const caseResults: EvalCaseResult[] = []; + + for (const evalCase of evalCases) { + caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge)); + } + + const summary = createSummary(caseResults); + const baseline = await loadBaselineFile(); + const report: EvalReport = { + generatedAt: new Date().toISOString(), + inputPath, + setName, + summary, + cases: caseResults, + }; + + if (baseline) { + report.baselineDelta = createBaselineDelta(summary, baseline.summary); + } + + const reportPath = await writeReport( + report, + options?.outputPath ?? buildDefaultReportPath(report.generatedAt), + ); + logger.info( + { + cases: summary.cases, + passRate: summary.passRate, + reportPath, + }, + "[Eval] run completed", + ); + + return { report, reportPath }; +} + +export async function findLatestEvalReportPath(): Promise { + const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR); + if (!existsSync(reportsDir)) { + return null; + } + + const entries = await readdir(reportsDir, { withFileTypes: true }); + const candidates = entries + .filter((entry) => entry.isFile()) + .map((entry) => entry.name) + .filter((name) => name.endsWith(".json") && name !== "baseline.json") + .sort((left, right) => right.localeCompare(left)); + + if (candidates.length === 0) { + return null; + } + + const latest = candidates[0]; + if (!latest) { + return null; + } + + return resolve(reportsDir, latest); +} + +export async function updateEvalBaseline( + reportPath?: string, +): Promise<{ + baseline: EvalBaselineFile; + baselinePath: string; + storedInRedis: boolean; +}> { + const sourcePath = + reportPath?.trim() ? resolve(process.cwd(), reportPath) : await findLatestEvalReportPath(); + if (!sourcePath) { + throw new Error("No eval report found to baseline"); + } + + const raw = await readFile(sourcePath, "utf8"); + const report = JSON.parse(raw) as EvalReport; + const baseline: EvalBaselineFile = { + reportPath: sourcePath, + setName: report.setName, + summary: report.summary, + updatedAt: new Date().toISOString(), + }; + + const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE); + await mkdir(dirname(baselinePath), { recursive: true }); + await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8"); + + const storedInRedis = await persistBaselineInRedis(baseline); + + return { baseline, baselinePath, storedInRedis }; +} + +async function persistBaselineInRedis( + baseline: EvalBaselineFile, +): Promise { + try { + const { GarnetClient } = await import("@common/garnet"); + const garnetClient = new GarnetClient(); + await garnetClient.connect(); + try { + await garnetClient.setString( + EVAL_BASELINE_KEY, + JSON.stringify(baseline), + ); + return true; + } finally { + await garnetClient.close(); + } + } catch { + return false; + } +} + +export async function publishEvalReportIfPossible( + report: EvalReport, +): Promise { + try { + const natsClient = new NatsClient(); + await natsClient.connect(); + try { + await natsClient.publish(AGENT_EVAL_REPORT_SUBJECT, report); + return true; + } finally { + await natsClient.close(); + } + } catch { + return false; + } +} diff --git a/src/eval/shadow-bus.test.ts b/src/eval/shadow-bus.test.ts new file mode 100644 index 0000000..60006f7 --- /dev/null +++ b/src/eval/shadow-bus.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { AgentRequest, AgentResponse } from "@agent/types"; +import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state"; +import { + ShadowEvalBus, + shouldSampleShadowEval, +} from "./shadow-bus.js"; + +const request: AgentRequest = { + requestId: "req-1", + skill: "onboarding", + chatroomId: "room-1", + userId: "user-1", + nickname: "프리리/95/여/D", + trigger: "user-message", + userMessage: "이제 뭐 남았지", +}; + +const response: AgentResponse = { + requestId: "req-1", + text: "이제 방 하트 인증만 남았어요.", + confidence: 0.9, + source: "llm", +}; + +const classified: ClassifiedIntent = { + intent: { kind: "status-question" }, + confidence: 0.98, + source: "regex", +}; + +const derived: DerivedState = { + phase: { kind: "photo-pending" }, + intent: { kind: "status-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 128, + allowShortResponse: true, + tier: 0, +}; + +describe("shouldSampleShadowEval", () => { + it("samples when the random value is under the tier rate", () => { + expect(shouldSampleShadowEval(0, 0)).toBe(true); + }); + + it("does not sample when the random value exceeds the tier rate", () => { + expect(shouldSampleShadowEval(0, 1)).toBe(false); + }); +}); + +describe("ShadowEvalBus", () => { + it("publishes a shadow-eval stream entry when sampled", () => { + const appendStream = vi.fn().mockResolvedValue("1-0"); + const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0); + + new ShadowEvalBus({ appendStream }).publish({ + classified, + createdAt: "2026-04-16T00:00:00.000Z", + derived, + request, + response, + }); + + expect(appendStream).toHaveBeenCalledTimes(1); + expect(appendStream.mock.calls[0]?.[0]).toBe("lolympus-agent:background"); + expect(appendStream.mock.calls[0]?.[1]).toMatchObject({ + type: "shadow-eval", + requestId: "req-1", + tier: "0", + }); + + randomSpy.mockRestore(); + }); + + it("skips publishing for empty responses", () => { + const appendStream = vi.fn().mockResolvedValue("1-0"); + + new ShadowEvalBus({ appendStream }).publish({ + classified, + createdAt: "2026-04-16T00:00:00.000Z", + derived, + request, + response: { + ...response, + text: "", + }, + }); + + expect(appendStream).not.toHaveBeenCalled(); + }); +}); diff --git a/src/eval/shadow-bus.ts b/src/eval/shadow-bus.ts new file mode 100644 index 0000000..971589a --- /dev/null +++ b/src/eval/shadow-bus.ts @@ -0,0 +1,90 @@ +import type { AgentRequest, AgentResponse } from "@agent/types"; +import type { GarnetClient } from "@common/garnet"; +import { getRequestLogger } from "@obs/context"; +import type { + ClassifiedIntent, + DerivedState, +} from "@skills/onboarding/state/onboarding-state"; +import { + getBackgroundStreamMaxLen, + getShadowEvalSamplingRate, +} from "../config.js"; + +const BACKGROUND_STREAM_KEY = "lolympus-agent:background"; + +export interface ShadowEvalEnvelope { + classified: ClassifiedIntent; + createdAt: string; + derived: DerivedState; + latencyMs?: number | undefined; + request: AgentRequest; + response: AgentResponse; +} + +export interface ShadowEvalPublisher { + publish(envelope: ShadowEvalEnvelope): void; +} + +export const noopShadowEvalPublisher: ShadowEvalPublisher = { + publish(): void {}, +}; + +export function shouldSampleShadowEval( + tier: 0 | 1 | 2, + randomValue = Math.random(), +): boolean { + return randomValue < getShadowEvalSamplingRate(tier); +} + +export class ShadowEvalBus implements ShadowEvalPublisher { + constructor( + private readonly garnetClient: Pick, + ) {} + + publish(envelope: ShadowEvalEnvelope): void { + if (!envelope.response.text.trim()) { + return; + } + + if (!shouldSampleShadowEval(envelope.derived.tier)) { + return; + } + + const logger = getRequestLogger(); + + void this.garnetClient + .appendStream( + BACKGROUND_STREAM_KEY, + { + type: "shadow-eval", + requestId: envelope.request.requestId, + chatroomId: envelope.request.chatroomId, + userId: envelope.request.userId, + tier: String(envelope.derived.tier), + latencyMs: + envelope.latencyMs !== undefined + ? String(envelope.latencyMs) + : "", + input: JSON.stringify({ + request: envelope.request, + }), + output: JSON.stringify(envelope.response), + derived: JSON.stringify(envelope.derived), + classified: JSON.stringify(envelope.classified), + createdAt: envelope.createdAt, + }, + { + maxLen: getBackgroundStreamMaxLen(), + }, + ) + .catch((error) => { + logger.warn( + { + err: error, + requestId: envelope.request.requestId, + }, + "[ShadowEvalBus] Failed to publish shadow eval job", + ); + }); + } +} diff --git a/src/eval/shadow-worker.test.ts b/src/eval/shadow-worker.test.ts new file mode 100644 index 0000000..e3b3afe --- /dev/null +++ b/src/eval/shadow-worker.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; + +import { + calculateEvalScoreDropPct, + calculateLlmErrorRate, + calculatePercentile, +} from "./shadow-worker.js"; + +describe("calculatePercentile", () => { + it("returns null for empty input", () => { + expect(calculatePercentile([], 0.95)).toBeNull(); + }); + + it("calculates the p95 from sorted values", () => { + expect(calculatePercentile([100, 200, 300, 400, 500], 0.95)).toBe(500); + }); +}); + +describe("calculateEvalScoreDropPct", () => { + it("returns null without a valid baseline", () => { + expect(calculateEvalScoreDropPct(4.2, null)).toBeNull(); + expect(calculateEvalScoreDropPct(4.2, 0)).toBeNull(); + }); + + it("computes percentage drop from baseline", () => { + expect(calculateEvalScoreDropPct(3.6, 4.0)).toBeCloseTo(10); + }); +}); + +describe("calculateLlmErrorRate", () => { + it("returns null when no observations are present", () => { + expect(calculateLlmErrorRate([])).toBeNull(); + }); + + it("computes error ratio across observations", () => { + expect( + calculateLlmErrorRate([ + { + llmJudgeErrored: true, + llmOverall: null, + recordedAt: 1, + responseLatencyMs: 100, + }, + { + llmJudgeErrored: false, + llmOverall: 4.2, + recordedAt: 2, + responseLatencyMs: 200, + }, + ]), + ).toBe(0.5); + }); +}); diff --git a/src/eval/shadow-worker.ts b/src/eval/shadow-worker.ts new file mode 100644 index 0000000..a5cdad4 --- /dev/null +++ b/src/eval/shadow-worker.ts @@ -0,0 +1,741 @@ +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import type { AgentRequest, AgentResponse } from "@agent/types"; +import type { + GarnetClient, + StreamEntry, + StreamPendingEntry, +} from "@common/garnet"; +import { summarizeConversation } from "@conv/summarizer"; +import type { LlmClient } from "@llm/llm-client"; +import { getLogger } from "@obs/logger"; +import type { AlertSender } from "@obs/alert"; +import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator"; +import type { + ClassifiedIntent, + DerivedState, +} from "@skills/onboarding/state/onboarding-state"; +import type { ConversationStore } from "@conv/conversation-store"; +import { + getAlertThresholdEvalScoreDropPct, + getAlertThresholdLlmErrorRate, + getAlertThresholdP95LatencyMs, + getEvalStreamMaxLen, +} from "../config.js"; +import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; +import type { + EvalBaselineFile, + EvalCaseDefinition, + EvalDerivedSnapshot, +} from "./types.js"; + +const BACKGROUND_STREAM_KEY = "lolympus-agent:background"; +const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq"; +const BACKGROUND_GROUP = "bg-worker"; +const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline"; +const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results"; +const EVAL_BASELINE_FILE = "eval-reports/baseline.json"; +const ALERT_WINDOW_MS = 5 * 60_000; +const ALERT_WINDOW_LABEL = "5m"; +const ALERT_TICK_MS = 60_000; +const MAX_DELIVERIES = 3; +const MIN_ERROR_RATE_OBSERVATIONS = 5; +const MIN_LATENCY_OBSERVATIONS = 5; +const MIN_SCORE_OBSERVATIONS = 3; +const PENDING_IDLE_MS = 1_000; +const READ_BLOCK_MS = 5_000; +const READ_COUNT = 10; +const PENDING_BATCH_SIZE = 20; +const STOP_TIMEOUT_MS = 10_000; + +type BackgroundJobType = "shadow-eval" | "summarize"; + +interface ShadowEvalJob { + classified: ClassifiedIntent; + createdAt: string; + derived: DerivedState; + latencyMs?: number | undefined; + request: AgentRequest; + response: AgentResponse; +} + +interface EvalObservation { + llmJudgeErrored: boolean; + llmOverall: number | null; + recordedAt: number; + responseLatencyMs: number | null; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +function promiseWithTimeout( + promise: Promise, + timeoutMs: number, +): Promise { + return Promise.race([ + promise, + new Promise((_, reject) => { + setTimeout(() => { + reject(new Error(`Timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }), + ]); +} + +function getJobType(message: StreamEntry): string { + return message.message["type"] ?? "unknown"; +} + +function parseOptionalNumberField(value: string | undefined): number | undefined { + if (!value?.trim()) { + return undefined; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function parseJsonField( + value: string | undefined, + fieldName: string, +): T { + if (!value?.trim()) { + throw new Error(`Missing ${fieldName} field`); + } + + return JSON.parse(value) as T; +} + +function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot { + return { + allowShortResponse: derived.allowShortResponse, + intent: derived.intent.kind, + phase: derived.phase.kind, + responseMode: derived.responseMode, + shouldRespond: derived.shouldRespond, + suggestedMaxTokens: derived.suggestedMaxTokens, + tier: derived.tier, + }; +} + +function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition { + return { + id: `shadow:${messageId}`, + lineNumber: 0, + input: { + request: job.request, + }, + expected: {}, + }; +} + +function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob { + const input = parseJsonField<{ request: AgentRequest }>( + message.message["input"], + "input", + ); + + return { + classified: parseJsonField( + message.message["classified"], + "classified", + ), + createdAt: message.message["createdAt"] ?? new Date().toISOString(), + derived: parseJsonField( + message.message["derived"], + "derived", + ), + latencyMs: parseOptionalNumberField(message.message["latencyMs"]), + request: input.request, + response: parseJsonField( + message.message["output"], + "output", + ), + }; +} + +export function calculatePercentile( + values: number[], + percentile: number, +): number | null { + if (values.length === 0) { + return null; + } + + const sorted = [...values].sort((left, right) => left - right); + const index = Math.min( + sorted.length - 1, + Math.max(0, Math.ceil(sorted.length * percentile) - 1), + ); + const selected = sorted[index]; + return selected === undefined ? null : selected; +} + +export function calculateEvalScoreDropPct( + currentAverage: number | null, + baselineAverage: number | null, +): number | null { + if ( + currentAverage === null || + baselineAverage === null || + baselineAverage <= 0 + ) { + return null; + } + + return ((baselineAverage - currentAverage) / baselineAverage) * 100; +} + +export function calculateLlmErrorRate( + observations: EvalObservation[], +): number | null { + if (observations.length === 0) { + return null; + } + + const errorCount = observations.filter( + (observation) => observation.llmJudgeErrored, + ).length; + return errorCount / observations.length; +} + +export class ShadowWorker { + private lastAggregateAt = 0; + private loopPromise: Promise | null = null; + private readonly logger = getLogger(); + private readonly recentEvalObservations: EvalObservation[] = []; + private stopRequested = false; + + constructor( + private readonly garnetClient: GarnetClient, + private readonly llmClient: LlmClient, + private readonly conversationStore: ConversationStore, + private readonly alertSender?: AlertSender, + private readonly consumerName = `bg-worker-${process.pid}`, + ) {} + + async start(): Promise { + await this.ensureConsumerGroup(); + + if (this.loopPromise) { + return; + } + + this.stopRequested = false; + this.loopPromise = this.runLoop(); + } + + async stop(): Promise { + this.stopRequested = true; + + if (!this.loopPromise) { + return; + } + + try { + await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS); + } catch (error) { + this.logger.warn( + { err: error }, + "[ShadowWorker] Stop timed out while waiting for the loop to finish", + ); + } finally { + this.loopPromise = null; + } + } + + private async ensureConsumerGroup(): Promise { + try { + await this.garnetClient.createConsumerGroup( + BACKGROUND_STREAM_KEY, + BACKGROUND_GROUP, + ); + } catch (error) { + if (!String(error).includes("BUSYGROUP")) { + throw error; + } + } + } + + private async runLoop(): Promise { + while (!this.stopRequested) { + try { + await this.runPeriodicChecksIfNeeded(); + + const recoveredPending = await this.recoverPendingOnce(); + if (recoveredPending) { + continue; + } + + const streams = await this.garnetClient.readGroup( + BACKGROUND_STREAM_KEY, + BACKGROUND_GROUP, + this.consumerName, + ">", + { + blockMs: READ_BLOCK_MS, + count: READ_COUNT, + }, + ); + + if (!streams) { + continue; + } + + for (const stream of streams) { + for (const message of stream.messages) { + if (this.stopRequested) { + return; + } + + await this.processMessage( + message, + message.deliveriesCounter ?? 1, + ); + } + } + } catch (error) { + this.logger.error( + { err: error }, + "[ShadowWorker] Loop iteration failed", + ); + await delay(1_000); + } + } + } + + private async recoverPendingOnce(): Promise { + const pending = await this.garnetClient.getPendingStreamRange( + BACKGROUND_STREAM_KEY, + BACKGROUND_GROUP, + PENDING_BATCH_SIZE, + ); + const recoverable = pending.filter( + (entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS, + ); + + if (recoverable.length === 0) { + return false; + } + + for (const entry of recoverable) { + if (this.stopRequested) { + break; + } + + await this.recoverPendingEntry(entry); + } + + return true; + } + + private async recoverPendingEntry( + entry: StreamPendingEntry, + ): Promise { + const nextDeliveryCount = entry.deliveriesCounter + 1; + const claimed = await this.garnetClient.claimPendingStream( + BACKGROUND_STREAM_KEY, + BACKGROUND_GROUP, + this.consumerName, + PENDING_IDLE_MS, + entry.id, + { + retryCount: nextDeliveryCount, + }, + ); + const message = claimed[0]; + + if (!message) { + return; + } + + if (nextDeliveryCount > MAX_DELIVERIES) { + await this.handleMaxDeliveries( + message, + nextDeliveryCount, + "recovered entry exceeded max deliveries", + ); + return; + } + + await this.processMessage(message, nextDeliveryCount); + } + + private async processMessage( + message: StreamEntry, + deliveryCount: number, + ): Promise { + try { + switch (getJobType(message) as BackgroundJobType) { + case "shadow-eval": + await this.processShadowEval(message); + await this.acknowledge(message.id); + return; + case "summarize": + await this.processSummarize(message); + await this.acknowledge(message.id); + return; + default: + this.logger.warn( + { + messageId: message.id, + type: getJobType(message), + }, + "[ShadowWorker] Unknown background job type; acknowledging", + ); + await this.acknowledge(message.id); + } + } catch (error) { + this.logger.warn( + { + err: error, + deliveryCount, + messageId: message.id, + type: getJobType(message), + }, + "[ShadowWorker] Job processing failed", + ); + + if (deliveryCount >= MAX_DELIVERIES) { + await this.handleMaxDeliveries( + message, + deliveryCount, + error instanceof Error ? error.message : String(error), + ); + } + } + } + + private async processShadowEval(message: StreamEntry): Promise { + const job = parseShadowEvalJob(message); + const validation = validateOnboardingResponse( + job.response.text, + job.derived, + { + allowShortResponse: job.derived.allowShortResponse, + }, + ); + const llmJudge = await judgeOnboardingWithLlm( + this.llmClient, + toEvalCase(job, message.id), + job.response.text, + toDerivedSnapshot(job.derived), + ); + const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall; + const llmJudgeErrored = + llmJudge.skipped && + llmJudge.reason !== "empty-response" && + llmJudge.reason !== "disabled"; + + this.recordEvalObservation({ + llmJudgeErrored, + llmOverall, + recordedAt: Date.now(), + responseLatencyMs: job.latencyMs ?? null, + }); + + await this.garnetClient.appendStream( + EVAL_RESULTS_STREAM_KEY, + { + type: "shadow-eval-result", + requestId: job.request.requestId, + chatroomId: job.request.chatroomId, + userId: job.request.userId, + tier: String(job.derived.tier), + intent: job.derived.intent.kind, + responseMode: job.derived.responseMode, + validatorStatus: validation.status, + latencyMs: + job.latencyMs !== undefined ? String(job.latencyMs) : "", + llmOverall: + llmOverall === null ? "" : String(llmOverall), + input: JSON.stringify({ + request: job.request, + }), + output: JSON.stringify(job.response), + derived: JSON.stringify(job.derived), + classified: JSON.stringify(job.classified), + validation: JSON.stringify(validation), + llmJudge: JSON.stringify(llmJudge), + evaluatedAt: new Date().toISOString(), + }, + { + maxLen: getEvalStreamMaxLen(), + }, + ); + } + + private async processSummarize(message: StreamEntry): Promise { + const chatroomId = message.message["chatroomId"]; + const userId = message.message["userId"]; + + if (!chatroomId?.trim() || !userId?.trim()) { + throw new Error("Missing summarize job identifiers"); + } + + const [history, existingSummary] = await Promise.all([ + this.conversationStore.getHistory(chatroomId, userId), + this.conversationStore.getSummary(chatroomId, userId), + ]); + + if (history.length === 0) { + return; + } + + const summary = await summarizeConversation( + this.llmClient, + history, + existingSummary ?? undefined, + ); + + await this.conversationStore.setSummary(chatroomId, userId, summary); + } + + private async handleMaxDeliveries( + message: StreamEntry, + deliveryCount: number, + lastError: string, + ): Promise { + const type = getJobType(message); + + if (type === "summarize") { + this.logger.warn( + { + deliveryCount, + lastError, + messageId: message.id, + }, + "[ShadowWorker] Dropping summarize job after max retries", + ); + await this.acknowledge(message.id); + return; + } + + await this.garnetClient.appendStream( + BACKGROUND_DLQ_STREAM_KEY, + { + type, + payload: JSON.stringify(message.message), + retryCount: String(deliveryCount), + lastError: lastError.slice(0, 500), + originalId: message.id, + }, + { + maxLen: 1_000, + }, + ); + await this.acknowledge(message.id); + } + + private acknowledge(id: string): Promise { + return this.garnetClient.acknowledgeStream( + BACKGROUND_STREAM_KEY, + BACKGROUND_GROUP, + id, + ); + } + + private recordEvalObservation(observation: EvalObservation): void { + this.recentEvalObservations.push(observation); + this.pruneOldObservations(Date.now()); + } + + private pruneOldObservations(now: number): void { + const cutoff = now - ALERT_WINDOW_MS; + let removeCount = 0; + + for (const observation of this.recentEvalObservations) { + if (observation.recordedAt >= cutoff) { + break; + } + removeCount += 1; + } + + if (removeCount > 0) { + this.recentEvalObservations.splice(0, removeCount); + } + } + + private async runPeriodicChecksIfNeeded(): Promise { + if (!this.alertSender) { + return; + } + + const now = Date.now(); + if (now - this.lastAggregateAt < ALERT_TICK_MS) { + return; + } + + this.lastAggregateAt = now; + this.pruneOldObservations(now); + + await this.checkLatencyAlert(now); + await this.checkEvalScoreDropAlert(now); + await this.checkLlmErrorRateAlert(now); + await this.checkOpenCircuitAlerts(now); + } + + private async checkLatencyAlert(now: number): Promise { + const latencySamples = this.recentEvalObservations.flatMap((observation) => + observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs], + ); + if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) { + return; + } + + const p95LatencyMs = calculatePercentile(latencySamples, 0.95); + const threshold = getAlertThresholdP95LatencyMs(); + if (p95LatencyMs === null || p95LatencyMs <= threshold) { + return; + } + + await this.alertSender.sendAlert({ + kind: "latency", + metric: "shadow_eval.p95_latency_ms", + severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn", + suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.", + threshold, + timestamp: new Date(now).toISOString(), + value: p95LatencyMs, + window: ALERT_WINDOW_LABEL, + }); + } + + private async checkEvalScoreDropAlert(now: number): Promise { + if (!this.alertSender) { + return; + } + + const scoreSamples = this.recentEvalObservations.flatMap((observation) => + observation.llmOverall === null ? [] : [observation.llmOverall], + ); + if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) { + return; + } + + const currentAverage = + scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length; + const baselineAverage = await this.loadBaselineOverallScore(); + const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage); + const threshold = getAlertThresholdEvalScoreDropPct(); + + if (dropPct === null || dropPct <= threshold) { + return; + } + + await this.alertSender.sendAlert({ + kind: "eval-score-drop", + metric: "shadow_eval.llm_overall_drop_pct", + severity: dropPct >= threshold * 1.5 ? "critical" : "warn", + suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.", + threshold, + timestamp: new Date(now).toISOString(), + value: Number(dropPct.toFixed(3)), + window: ALERT_WINDOW_LABEL, + }); + } + + private async checkLlmErrorRateAlert(now: number): Promise { + if ( + !this.alertSender || + this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS + ) { + return; + } + + const errorRate = calculateLlmErrorRate(this.recentEvalObservations); + const threshold = getAlertThresholdLlmErrorRate(); + if (errorRate === null || errorRate <= threshold) { + return; + } + + await this.alertSender.sendAlert({ + kind: "llm-error-rate", + metric: "shadow_eval.llm_error_rate", + severity: errorRate >= threshold * 1.5 ? "critical" : "warn", + suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.", + threshold, + timestamp: new Date(now).toISOString(), + value: Number(errorRate.toFixed(4)), + window: ALERT_WINDOW_LABEL, + }); + } + + private async checkOpenCircuitAlerts(now: number): Promise { + if (!this.alertSender) { + return; + } + + const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*"); + + for (const key of circuitKeys) { + const record = await this.garnetClient.getHash(key); + if (record["state"] !== "open") { + continue; + } + + const endpointHash = key.split(":").at(-1) ?? "unknown"; + const failureCount = Number(record["failureCount"]); + + await this.alertSender.sendAlert( + { + kind: "endpoint-down", + metric: "llm.circuit.open", + severity: "critical", + suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.", + threshold: 0, + timestamp: new Date(now).toISOString(), + value: Number.isFinite(failureCount) ? failureCount : 1, + window: ALERT_WINDOW_LABEL, + }, + { + dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`, + }, + ); + } + } + + private async loadBaselineOverallScore(): Promise { + const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY); + const redisParsed = parseBaselineOverallFromRaw(redisBaseline); + if (redisParsed !== null) { + return redisParsed; + } + + const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE); + if (!existsSync(filePath)) { + return null; + } + + try { + const fileRaw = await readFile(filePath, "utf8"); + return parseBaselineOverallFromRaw(fileRaw); + } catch (error) { + this.logger.warn( + { err: error, filePath }, + "[ShadowWorker] Failed to read baseline file", + ); + return null; + } + } +} + +function parseBaselineOverallFromRaw(raw: string | null): number | null { + if (!raw?.trim()) { + return null; + } + + try { + const parsed = JSON.parse(raw) as Partial; + const value = parsed.summary?.llmAverageOverall; + return typeof value === "number" ? value : null; + } catch { + return null; + } +} diff --git a/src/eval/types.ts b/src/eval/types.ts new file mode 100644 index 0000000..20d44e5 --- /dev/null +++ b/src/eval/types.ts @@ -0,0 +1,151 @@ +import type { + AgentRequest, + AgentResponse, + ConversationMessage, +} from "@agent/types"; +import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; +import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state"; + +export interface EvalRoomContextMessage { + senderId: string; + nickname: string; + message: string; + timestamp: string; +} + +export interface EvalCaseInput { + request: AgentRequest; + conversationHistory?: ConversationMessage[] | undefined; + roomContext?: EvalRoomContextMessage[] | undefined; + session?: OnboardingSession | null | undefined; + blackboard?: Blackboard | null | undefined; + summary?: string | undefined; +} + +export interface EvalCaseExpected { + intent?: string | undefined; + mustIncludeAny?: string[] | undefined; + mustNotInclude?: string[] | undefined; + responseMode?: string | undefined; +} + +export interface EvalCaseDefinition { + id: string; + lineNumber: number; + name?: string | undefined; + notes?: string | undefined; + input: EvalCaseInput; + expected: EvalCaseExpected; +} + +export interface RuleJudgeCheck { + name: + | "validator" + | "intent" + | "responseMode" + | "mustIncludeAny" + | "mustNotInclude"; + passed: boolean; + skipped: boolean; + detail?: string | undefined; +} + +export interface RuleJudgeResult { + passed: boolean; + score: number; + checks: RuleJudgeCheck[]; + validatorStatus: "pass" | "soft-warn" | "reject"; + validatorReasons: string[]; +} + +export interface LlmJudgeScores { + friendliness: number; + accuracy: number; + concision: number; + scope: number; + overall: number; +} + +export type LlmJudgeResult = + | { + skipped: true; + reason: string; + raw?: string | undefined; + } + | { + skipped: false; + model: string; + endpoint: string; + provider: string; + scores: LlmJudgeScores; + rationale: string; + raw: string; + }; + +export interface EvalDerivedSnapshot { + allowShortResponse: boolean; + intent: string; + phase: string; + responseMode: DerivedState["responseMode"]; + shouldRespond: boolean; + suggestedMaxTokens: number; + tier: DerivedState["tier"]; +} + +export interface EvalClassifiedSnapshot { + confidence: number; + intent: string; + source: ClassifiedIntent["source"]; +} + +export interface EvalCaseResult { + id: string; + lineNumber: number; + name?: string | undefined; + latencyMs: number; + classified: EvalClassifiedSnapshot; + derived: EvalDerivedSnapshot; + request: AgentRequest; + response: AgentResponse; + ruleJudge: RuleJudgeResult; + llmJudge: LlmJudgeResult; + error?: string | undefined; +} + +export interface EvalReportSummary { + averageLatencyMs: number; + cases: number; + emptyResponses: number; + llmAverageAccuracy: number | null; + llmAverageConcision: number | null; + llmAverageFriendliness: number | null; + llmAverageOverall: number | null; + llmAverageScope: number | null; + llmJudgedCases: number; + passRate: number; + passedCases: number; + ruleAverageScore: number; +} + +export interface EvalSummaryDelta { + averageLatencyMs: number | null; + llmAverageOverall: number | null; + passRate: number | null; + ruleAverageScore: number | null; +} + +export interface EvalReport { + baselineDelta?: EvalSummaryDelta | undefined; + generatedAt: string; + inputPath: string; + setName: string; + summary: EvalReportSummary; + cases: EvalCaseResult[]; +} + +export interface EvalBaselineFile { + reportPath: string; + setName: string; + summary: EvalReportSummary; + updatedAt: string; +} diff --git a/src/llm/index.ts b/src/llm/index.ts index a0845bf..9eb4f1c 100644 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -1,7 +1,10 @@ export type { + ChatCompletionOptions, + ChatCompletionResult, ChatCompletionRequest, ChatCompletionResponse, ChatMessage, + ChatToolCall, LlmClientOptions, LlmEndpointConfig, } from "./types.js"; diff --git a/src/llm/llm-client.ts b/src/llm/llm-client.ts index bad3dae..0263b1c 100644 --- a/src/llm/llm-client.ts +++ b/src/llm/llm-client.ts @@ -1,49 +1,37 @@ import type { - ChatCompletionRequest, - ChatCompletionResponse, + ChatCompletionOptions, ChatMessage, + ChatCompletionResult, + ChatCompletionResponse, LlmClientOptions, LlmEndpointConfig, + ProviderName, } from "./types.js"; import { getRequestLogger } from "@obs/context"; import { withLlmSpan } from "@obs/spans"; import { getLlmHealthTimeoutMs, - getLlmMaxTokens, getLlmRequestTimeoutMs, getLlmTemperature, } from "../config.js"; import { LlmCircuitBreaker } from "./circuit.js"; +import { getProviderAdapter } from "./providers/index.js"; +import { selectMaxTokensForTier, selectModelForProvider, selectProviderOrder } from "./routing.js"; const DEFAULT_MODEL = "Qwen3.5-4B"; -function buildCliProxyHealthUrl(endpoint: string): string { - try { - const url = new URL(endpoint); - url.pathname = "/"; - url.search = ""; - url.hash = ""; - return url.toString(); - } catch { - const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, ""); - return `${normalized.replace(/\/+$/u, "")}/`; - } -} - function normalizeEndpointConfig( - endpoint: string | { url: string; provider?: "default" | "cliproxy" }, + endpoint: string | { url: string; provider?: ProviderName | "default" | undefined }, ): LlmEndpointConfig { const raw = typeof endpoint === "string" - ? { provider: "default" as const, url: endpoint } - : { provider: endpoint.provider ?? "default", url: endpoint.url }; + ? { provider: "llama" as const, url: endpoint } + : { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url }; + const provider = getProviderAdapter(raw.provider); return { completionUrl: buildCompletionUrl(raw.url), - healthUrl: - raw.provider === "cliproxy" - ? buildCliProxyHealthUrl(raw.url) - : buildHealthUrl(raw.url), + healthUrl: provider.createHealthUrl(raw.url), provider: raw.provider, }; } @@ -65,49 +53,68 @@ function buildCompletionUrl(endpoint: string): string { } } -function buildHealthUrl(endpoint: string): string { - try { - const url = new URL(endpoint); - url.pathname = "/health"; - url.search = ""; - url.hash = ""; - return url.toString(); - } catch { - if (endpoint.includes("/v1/chat/completions")) { - return `${endpoint.split("/v1/chat/completions")[0]}/health`; - } - return `${endpoint.replace(/\/+$/u, "")}/health`; +function getRequestSignal( + timeoutMs: number, + externalSignal?: AbortSignal, +): AbortSignal { + const timeoutSignal = AbortSignal.timeout(timeoutMs); + + if (!externalSignal) { + return timeoutSignal; } + + if (typeof AbortSignal.any === "function") { + return AbortSignal.any([timeoutSignal, externalSignal]); + } + + const controller = new AbortController(); + const abort = (): void => controller.abort(); + + timeoutSignal.addEventListener("abort", abort, { once: true }); + externalSignal.addEventListener("abort", abort, { once: true }); + + return controller.signal; } export class LlmClient { private readonly circuit: LlmCircuitBreaker; private readonly endpoints: LlmEndpointConfig[]; private readonly apiKey: string | undefined; - private readonly model: string; + private readonly llamaModel: string; + private readonly providerApiKeys: Partial>; constructor(options: LlmClientOptions) { this.circuit = new LlmCircuitBreaker(options.garnetClient); this.endpoints = options.endpoints.map(normalizeEndpointConfig); this.apiKey = options.apiKey; - this.model = options.model ?? DEFAULT_MODEL; + this.llamaModel = options.model ?? DEFAULT_MODEL; + this.providerApiKeys = options.providerApiKeys ?? {}; } - private getHeaders(includeJson = false): Record { + private getHeaders( + provider: ProviderName, + includeJson = false, + ): Record { const headers: Record = {}; if (includeJson) { headers["Content-Type"] = "application/json"; } - if (this.apiKey) { - headers["Authorization"] = `Bearer ${this.apiKey}`; + const apiKey = this.providerApiKeys[provider] ?? this.apiKey; + if (apiKey) { + headers["Authorization"] = `Bearer ${apiKey}`; } return headers; } private async checkHealth(endpoint: LlmEndpointConfig): Promise { + const provider = getProviderAdapter(endpoint.provider); + if (!provider.supportsHealthCheck) { + return true; + } + try { const response = await fetch(endpoint.healthUrl, { - headers: this.getHeaders(), + headers: this.getHeaders(endpoint.provider), method: "GET", signal: AbortSignal.timeout(getLlmHealthTimeoutMs()), }); @@ -122,12 +129,8 @@ export class LlmClient { async chatCompletion( messages: ChatMessage[], - options?: { - jsonMode?: boolean | undefined; - maxTokens?: number | undefined; - temperature?: number | undefined; - }, - ): Promise<{ content: string; endpoint: string } | null> { + options?: ChatCompletionOptions, + ): Promise { const logger = getRequestLogger(); if (this.endpoints.length === 0) { @@ -135,18 +138,10 @@ export class LlmClient { return null; } - const request: ChatCompletionRequest = { - model: this.model, - messages, - max_tokens: options?.maxTokens ?? getLlmMaxTokens(), - temperature: options?.temperature ?? getLlmTemperature(), - }; + const orderedEndpoints = selectProviderOrder(this.endpoints, options); - if (options?.jsonMode) { - request.response_format = { type: "json_object" }; - } - - for (const endpoint of this.endpoints) { + for (const endpoint of orderedEndpoints) { + const provider = getProviderAdapter(endpoint.provider); const healthy = await this.circuit.canUse( endpoint.completionUrl, () => this.checkHealth(endpoint), @@ -155,7 +150,7 @@ export class LlmClient { logger.warn( { endpoint: endpoint.completionUrl, - provider: endpoint.provider ?? 'default', + provider: provider.name, }, "[LLM] Endpoint unavailable via circuit breaker", ); @@ -163,17 +158,30 @@ export class LlmClient { } try { + const requestTimeoutMs = Math.max( + 1, + Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY), + ); + const request = provider.prepareRequest({ + jsonMode: options?.jsonMode, + maxTokens: selectMaxTokensForTier(options), + messages, + model: selectModelForProvider(provider.name, options, this.llamaModel), + temperature: options?.temperature ?? getLlmTemperature(), + tools: options?.tools, + }); + const llmResult = await withLlmSpan( { "gen_ai.request.model": request.model, - "gen_ai.system": endpoint.provider ?? 'default', + "gen_ai.system": provider.name, }, async (span) => { const response = await fetch(endpoint.completionUrl, { body: JSON.stringify(request), - headers: this.getHeaders(true), + headers: this.getHeaders(provider.name, true), method: "POST", - signal: AbortSignal.timeout(getLlmRequestTimeoutMs()), + signal: getRequestSignal(requestTimeoutMs, options?.signal), }); if (!response.ok) { @@ -181,7 +189,7 @@ export class LlmClient { logger.warn( { endpoint: endpoint.completionUrl, - provider: endpoint.provider ?? 'default', + provider: provider.name, status: response.status, body, }, @@ -191,7 +199,7 @@ export class LlmClient { } const payload = (await response.json()) as ChatCompletionResponse; - const content = payload.choices?.[0]?.message?.content; + const result = provider.toResult(payload, endpoint.completionUrl); span.setAttributes({ "gen_ai.response.finish_reasons": payload.choices @@ -201,21 +209,18 @@ export class LlmClient { "gen_ai.usage.output_tokens": payload.usage?.completion_tokens, }); - if (typeof content !== "string" || !content.trim()) { + if (!result) { logger.warn( { endpoint: endpoint.completionUrl, - provider: endpoint.provider ?? 'default', + provider: provider.name, }, "[LLM] Empty response from endpoint", ); return null; } - return { - content: content.trim(), - endpoint: endpoint.completionUrl, - }; + return result; }, ); @@ -231,7 +236,7 @@ export class LlmClient { logger.warn( { endpoint: endpoint.completionUrl, - provider: endpoint.provider ?? 'default', + provider: provider.name, error: error instanceof Error ? error.message : String(error), }, "[LLM] Request error", diff --git a/src/llm/providers/cliproxy.ts b/src/llm/providers/cliproxy.ts new file mode 100644 index 0000000..25d0ead --- /dev/null +++ b/src/llm/providers/cliproxy.ts @@ -0,0 +1,30 @@ +import { buildOpenAiRequest, normalizeProviderResponse } from "./shared.js"; +import type { ProviderAdapter } from "./types.js"; + +export const cliproxyProvider: ProviderAdapter = { + name: "cliproxy", + capabilities: { + jsonMode: true, + promptCache: false, + toolCalls: "openai", + }, + createHealthUrl(endpoint: string): string { + try { + const url = new URL(endpoint); + url.pathname = "/"; + url.search = ""; + url.hash = ""; + return url.toString(); + } catch { + const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, ""); + return `${normalized.replace(/\/+$/u, "")}/`; + } + }, + prepareRequest(input) { + return buildOpenAiRequest(input); + }, + supportsHealthCheck: true, + toResult(payload, endpoint) { + return normalizeProviderResponse(payload, endpoint, "cliproxy"); + }, +}; diff --git a/src/llm/providers/index.ts b/src/llm/providers/index.ts new file mode 100644 index 0000000..5a54a23 --- /dev/null +++ b/src/llm/providers/index.ts @@ -0,0 +1,18 @@ +import type { ProviderName } from "../types.js"; +import { cliproxyProvider } from "./cliproxy.js"; +import { llamaProvider } from "./llama.js"; +import { openrouterProvider } from "./openrouter.js"; +import type { ProviderAdapter } from "./types.js"; + +const PROVIDERS: Record = { + cliproxy: cliproxyProvider, + llama: llamaProvider, + openrouter: openrouterProvider, +}; + +export function getProviderAdapter(name: ProviderName): ProviderAdapter { + return PROVIDERS[name]; +} + +export { cliproxyProvider, llamaProvider, openrouterProvider }; +export type { ProviderAdapter, ProviderRequestInput } from "./types.js"; diff --git a/src/llm/providers/llama.ts b/src/llm/providers/llama.ts new file mode 100644 index 0000000..30616c4 --- /dev/null +++ b/src/llm/providers/llama.ts @@ -0,0 +1,32 @@ +import { buildHermesRequest, normalizeProviderResponse } from "./shared.js"; +import type { ProviderAdapter } from "./types.js"; + +export const llamaProvider: ProviderAdapter = { + name: "llama", + capabilities: { + jsonMode: false, + promptCache: false, + toolCalls: "hermes", + }, + createHealthUrl(endpoint: string): string { + try { + const url = new URL(endpoint); + url.pathname = "/health"; + url.search = ""; + url.hash = ""; + return url.toString(); + } catch { + if (endpoint.includes("/v1/chat/completions")) { + return `${endpoint.split("/v1/chat/completions")[0]}/health`; + } + return `${endpoint.replace(/\/+$/u, "")}/health`; + } + }, + prepareRequest(input) { + return buildHermesRequest(input); + }, + supportsHealthCheck: true, + toResult(payload, endpoint) { + return normalizeProviderResponse(payload, endpoint, "llama"); + }, +}; diff --git a/src/llm/providers/openrouter.ts b/src/llm/providers/openrouter.ts new file mode 100644 index 0000000..c530b37 --- /dev/null +++ b/src/llm/providers/openrouter.ts @@ -0,0 +1,21 @@ +import { buildOpenAiRequest, normalizeProviderResponse } from "./shared.js"; +import type { ProviderAdapter } from "./types.js"; + +export const openrouterProvider: ProviderAdapter = { + name: "openrouter", + capabilities: { + jsonMode: true, + promptCache: false, + toolCalls: "openai", + }, + createHealthUrl(endpoint: string): string { + return endpoint; + }, + prepareRequest(input) { + return buildOpenAiRequest(input); + }, + supportsHealthCheck: false, + toResult(payload, endpoint) { + return normalizeProviderResponse(payload, endpoint, "openrouter"); + }, +}; diff --git a/src/llm/providers/shared.ts b/src/llm/providers/shared.ts new file mode 100644 index 0000000..e59abd7 --- /dev/null +++ b/src/llm/providers/shared.ts @@ -0,0 +1,235 @@ +import { serializeToolsForHermes, serializeToolsForOpenAi } from "@tools/serializer"; +import type { Tool } from "@tools/tool"; +import type { + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResult, + ChatMessage, + ChatToolCall, +} from "../types.js"; +import type { ProviderRequestInput } from "./types.js"; + +function parseJsonSafely(value: string): unknown { + try { + return JSON.parse(value) as unknown; + } catch { + return value; + } +} + +function serializeHermesToolCalls(toolCalls: readonly ChatToolCall[]): string { + const payload = toolCalls.map((toolCall) => ({ + id: toolCall.id, + name: toolCall.function.name, + arguments: parseJsonSafely(toolCall.function.arguments), + })); + + return `${JSON.stringify(payload)}`; +} + +function serializeHermesToolResult(message: ChatMessage): string { + return `${JSON.stringify({ + name: message.name ?? message.tool_call_id ?? "tool", + result: parseJsonSafely(message.content), + })}`; +} + +function injectHermesToolBlock( + messages: ChatMessage[], + tools: readonly Tool[] | undefined, +): ChatMessage[] { + if (!tools || tools.length === 0) { + return [...messages]; + } + + const toolBlock = serializeToolsForHermes(tools); + const cloned = messages.map((message) => ({ ...message })); + const firstSystemIndex = cloned.findIndex((message) => message.role === "system"); + + if (firstSystemIndex === -1) { + cloned.unshift({ + role: "system", + content: toolBlock, + }); + return cloned; + } + + const current = cloned[firstSystemIndex]; + if (!current) { + return cloned; + } + + if (current.content.includes("")) { + return cloned; + } + + cloned[firstSystemIndex] = { + ...current, + content: `${current.content}\n\n${toolBlock}`, + }; + return cloned; +} + +function injectJsonModeReminder(messages: ChatMessage[]): ChatMessage[] { + const cloned = messages.map((message) => ({ ...message })); + const firstSystemIndex = cloned.findIndex((message) => message.role === "system"); + const reminder = "반드시 JSON object 하나만 출력하세요. 설명 문장은 쓰지 마세요."; + + if (firstSystemIndex === -1) { + cloned.unshift({ + role: "system", + content: reminder, + }); + return cloned; + } + + const current = cloned[firstSystemIndex]; + if (!current || current.content.includes(reminder)) { + return cloned; + } + + cloned[firstSystemIndex] = { + ...current, + content: `${current.content}\n\n${reminder}`, + }; + return cloned; +} + +export function normalizeOpenAiMessages(messages: ChatMessage[]): ChatMessage[] { + return messages.map((message) => { + if (message.role === "tool") { + return { + role: "tool", + content: message.content, + tool_call_id: message.tool_call_id, + }; + } + + return { + role: message.role, + content: message.content, + tool_calls: message.tool_calls, + }; + }); +} + +export function normalizeHermesMessages(messages: ChatMessage[]): ChatMessage[] { + return messages.map((message) => { + if (message.role === "tool") { + return { + role: "user", + content: serializeHermesToolResult(message), + }; + } + + if (message.tool_calls && message.tool_calls.length > 0) { + const parts = [ + message.content.trim(), + serializeHermesToolCalls(message.tool_calls), + ].filter(Boolean); + return { + role: message.role, + content: parts.join("\n\n"), + }; + } + + return { + role: message.role, + content: message.content, + }; + }); +} + +export function buildOpenAiRequest( + input: ProviderRequestInput, +): ChatCompletionRequest { + const request: ChatCompletionRequest = { + model: input.model, + messages: normalizeOpenAiMessages(input.messages), + max_tokens: input.maxTokens, + temperature: input.temperature, + }; + + if (input.jsonMode) { + request.response_format = { type: "json_object" }; + } + + if (input.tools && input.tools.length > 0) { + request.tools = serializeToolsForOpenAi(input.tools); + } + + return request; +} + +export function buildHermesRequest( + input: ProviderRequestInput, +): ChatCompletionRequest { + let messages = injectHermesToolBlock(input.messages, input.tools); + messages = normalizeHermesMessages(messages); + + if (input.jsonMode) { + messages = injectJsonModeReminder(messages); + } + + return { + model: input.model, + messages, + max_tokens: input.maxTokens, + temperature: input.temperature, + }; +} + +function normalizeToolCalls( + toolCalls: ChatToolCall[] | null | undefined, +): ChatToolCall[] | undefined { + if (!toolCalls || toolCalls.length === 0) { + return undefined; + } + + const normalized = toolCalls.flatMap((toolCall) => { + const name = toolCall.function?.name?.trim(); + const args = toolCall.function?.arguments; + if (!name || typeof args !== "string") { + return []; + } + + return [ + { + id: toolCall.id, + type: "function" as const, + function: { + name, + arguments: args, + }, + }, + ]; + }); + + return normalized.length > 0 ? normalized : undefined; +} + +export function normalizeProviderResponse( + payload: ChatCompletionResponse, + endpoint: string, + provider: ChatCompletionResult["provider"], +): ChatCompletionResult | null { + const firstMessage = payload.choices?.[0]?.message; + const content = firstMessage?.content?.trim() ?? ""; + const toolCalls = normalizeToolCalls(firstMessage?.tool_calls); + + if (!content && (!toolCalls || toolCalls.length === 0)) { + return null; + } + + return { + content, + endpoint, + message: { + role: firstMessage?.role ?? "assistant", + content, + tool_calls: toolCalls, + }, + provider, + raw: payload, + }; +} diff --git a/src/llm/providers/types.ts b/src/llm/providers/types.ts new file mode 100644 index 0000000..28601f3 --- /dev/null +++ b/src/llm/providers/types.ts @@ -0,0 +1,33 @@ +import type { Tool } from "@tools/tool"; +import type { + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResult, + ChatMessage, + ProviderName, +} from "../types.js"; + +export interface ProviderRequestInput { + jsonMode?: boolean | undefined; + maxTokens: number; + messages: ChatMessage[]; + model: string; + temperature: number; + tools?: readonly Tool[] | undefined; +} + +export interface ProviderAdapter { + capabilities: { + jsonMode: boolean; + promptCache: false; + toolCalls: "openai" | "hermes" | "none"; + }; + createHealthUrl(endpoint: string): string; + name: ProviderName; + prepareRequest(input: ProviderRequestInput): ChatCompletionRequest; + supportsHealthCheck: boolean; + toResult( + payload: ChatCompletionResponse, + endpoint: string, + ): ChatCompletionResult | null; +} diff --git a/src/llm/routing.test.ts b/src/llm/routing.test.ts new file mode 100644 index 0000000..513ea58 --- /dev/null +++ b/src/llm/routing.test.ts @@ -0,0 +1,102 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { LlmEndpointConfig } from "./types.js"; +import { + selectMaxTokensForTier, + selectModelForProvider, + selectProviderOrder, +} from "./routing.js"; + +describe("llm routing", () => { + afterEach(() => { + delete process.env["OPENROUTER_ENABLED"]; + delete process.env["OPENROUTER_TIER1_MODEL"]; + delete process.env["OPENROUTER_TIER2_MODEL"]; + delete process.env["TIER0_MODEL"]; + delete process.env["TIER1_PLANNER_MODEL"]; + delete process.env["TIER1_RESPONDER_MODEL"]; + delete process.env["TIER2_CRITIC_MODEL"]; + vi.restoreAllMocks(); + }); + + it("prefers cliproxy then llama for tier 0 even when openrouter is enabled", () => { + process.env["OPENROUTER_ENABLED"] = "true"; + const endpoints: LlmEndpointConfig[] = [ + { + completionUrl: "http://llama.test/v1/chat/completions", + healthUrl: "http://llama.test/health", + provider: "llama", + }, + { + completionUrl: "http://openrouter.test/v1/chat/completions", + healthUrl: "http://openrouter.test", + provider: "openrouter", + }, + { + completionUrl: "http://cliproxy.test/v1/chat/completions", + healthUrl: "http://cliproxy.test", + provider: "cliproxy", + }, + ]; + + expect( + selectProviderOrder(endpoints, { tier: 0 }).map((endpoint) => endpoint.provider), + ).toEqual(["cliproxy", "llama"]); + }); + + it("includes openrouter between cliproxy and llama for tier 1+", () => { + process.env["OPENROUTER_ENABLED"] = "true"; + const endpoints: LlmEndpointConfig[] = [ + { + completionUrl: "http://llama.test/v1/chat/completions", + healthUrl: "http://llama.test/health", + provider: "llama", + }, + { + completionUrl: "http://openrouter.test/v1/chat/completions", + healthUrl: "http://openrouter.test", + provider: "openrouter", + }, + { + completionUrl: "http://cliproxy.test/v1/chat/completions", + healthUrl: "http://cliproxy.test", + provider: "cliproxy", + }, + ]; + + expect( + selectProviderOrder(endpoints, { tier: 1 }).map((endpoint) => endpoint.provider), + ).toEqual(["cliproxy", "openrouter", "llama"]); + }); + + it("selects role-aware models per provider", () => { + process.env["TIER1_PLANNER_MODEL"] = "cliproxy-planner"; + process.env["TIER2_CRITIC_MODEL"] = "cliproxy-critic"; + process.env["OPENROUTER_TIER1_MODEL"] = "openrouter-tier1"; + process.env["OPENROUTER_TIER2_MODEL"] = "openrouter-tier2"; + + expect( + selectModelForProvider("cliproxy", { role: "planner", tier: 1 }, "local-qwen"), + ).toBe("cliproxy-planner"); + expect( + selectModelForProvider("cliproxy", { role: "critic", tier: 2 }, "local-qwen"), + ).toBe("cliproxy-critic"); + expect(selectModelForProvider("openrouter", { tier: 1 }, "local-qwen")).toBe( + "openrouter-tier1", + ); + expect( + selectModelForProvider("openrouter", { role: "critic", tier: 2 }, "local-qwen"), + ).toBe("openrouter-tier2"); + expect(selectModelForProvider("llama", { tier: 2 }, "local-qwen")).toBe("local-qwen"); + }); + + it("chooses tier defaults for max tokens when explicit maxTokens is absent", () => { + process.env["TIER0_MAX_TOKENS"] = "111"; + process.env["TIER1_MAX_TOKENS"] = "222"; + process.env["TIER2_MAX_TOKENS"] = "333"; + + expect(selectMaxTokensForTier({ tier: 0 })).toBe(111); + expect(selectMaxTokensForTier({ tier: 1 })).toBe(222); + expect(selectMaxTokensForTier({ tier: 2 })).toBe(333); + expect(selectMaxTokensForTier({ tier: 1, maxTokens: 77 })).toBe(77); + }); +}); diff --git a/src/llm/routing.ts b/src/llm/routing.ts new file mode 100644 index 0000000..d86c687 --- /dev/null +++ b/src/llm/routing.ts @@ -0,0 +1,99 @@ +import { + getOpenRouterEnabled, + getOpenRouterTier1Model, + getOpenRouterTier2Model, + getTier0MaxTokens, + getTier0Model, + getTier1MaxTokens, + getTier1PlannerModel, + getTier1ResponderModel, + getTier2CriticModel, + getTier2MaxTokens, + getLlmMaxTokens, +} from "../config.js"; +import type { ChatCompletionOptions, LlmEndpointConfig, ProviderName } from "./types.js"; + +export function selectModelForProvider( + provider: ProviderName, + options: ChatCompletionOptions | undefined, + llamaFallbackModel: string, +): string { + if (options?.modelOverride?.trim()) { + return options.modelOverride.trim(); + } + + if (provider === "llama") { + return llamaFallbackModel; + } + + if (provider === "openrouter") { + return options?.role === "critic" + ? getOpenRouterTier2Model() + : getOpenRouterTier1Model(); + } + + if (options?.role === "planner") { + return getTier1PlannerModel(); + } + + if (options?.role === "critic") { + return getTier2CriticModel(); + } + + if (options?.tier === 1 || options?.tier === 2) { + return getTier1ResponderModel(); + } + + return getTier0Model(); +} + +export function selectMaxTokensForTier( + options: ChatCompletionOptions | undefined, +): number { + if (options?.maxTokens !== undefined) { + return options.maxTokens; + } + + if (options?.tier === 0) { + return getTier0MaxTokens(); + } + + if (options?.tier === 1) { + return getTier1MaxTokens(); + } + + if (options?.tier === 2) { + return getTier2MaxTokens(); + } + + return getLlmMaxTokens(); +} + +export function selectProviderOrder( + endpoints: readonly LlmEndpointConfig[], + options: ChatCompletionOptions | undefined, +): LlmEndpointConfig[] { + const preferredProviders: ProviderName[] = ["cliproxy"]; + + if (getOpenRouterEnabled() && options?.tier !== 0) { + preferredProviders.push("openrouter"); + } + + preferredProviders.push("llama"); + + const weighted = endpoints.map((endpoint, index) => ({ + endpoint, + index, + priority: preferredProviders.indexOf(endpoint.provider), + })); + + return weighted + .filter((item) => item.priority !== -1) + .sort((left, right) => { + if (left.priority !== right.priority) { + return left.priority - right.priority; + } + return left.index - right.index; + }) + .map((item) => item.endpoint); +} diff --git a/src/llm/types.ts b/src/llm/types.ts index a1176f7..fece76a 100644 --- a/src/llm/types.ts +++ b/src/llm/types.ts @@ -1,8 +1,23 @@ import type { GarnetClient } from '@common/garnet'; +import type { Tool } from '@tools/tool'; + +export type ProviderName = "cliproxy" | "llama" | "openrouter"; + +export interface ChatToolCall { + id?: string | undefined; + type: "function"; + function: { + name: string; + arguments: string; + }; +} export interface ChatMessage { - role: "system" | "user" | "assistant"; + role: "system" | "user" | "assistant" | "tool"; content: string; + name?: string | undefined; + tool_call_id?: string | undefined; + tool_calls?: ChatToolCall[] | undefined; } export interface ChatCompletionRequest { @@ -11,6 +26,7 @@ export interface ChatCompletionRequest { max_tokens: number; temperature: number; response_format?: { type: string } | undefined; + tools?: unknown[] | undefined; } export interface ChatCompletionResponse { @@ -18,7 +34,9 @@ export interface ChatCompletionResponse { choices?: Array<{ finish_reason?: string | null; message?: { + role?: "assistant" | "tool" | "user" | "system" | null; content?: string | null; + tool_calls?: ChatToolCall[] | null; }; }>; usage?: { @@ -28,15 +46,38 @@ export interface ChatCompletionResponse { }; } +export interface ChatCompletionResult { + content: string; + endpoint: string; + message: ChatMessage; + provider: ProviderName; + raw: ChatCompletionResponse; +} + +export interface ChatCompletionOptions { + deadlineMs?: number | undefined; + jsonMode?: boolean | undefined; + maxTokens?: number | undefined; + modelOverride?: string | undefined; + role?: "planner" | "responder" | "critic" | undefined; + signal?: AbortSignal | undefined; + temperature?: number | undefined; + tier?: 0 | 1 | 2 | undefined; + tools?: readonly Tool[] | undefined; +} + export interface LlmEndpointConfig { completionUrl: string; healthUrl: string; - provider?: "default" | "cliproxy" | undefined; + provider: ProviderName; } export interface LlmClientOptions { - endpoints: Array; + endpoints: Array< + string | { url: string; provider?: ProviderName | "default" | undefined } + >; garnetClient?: GarnetClient | undefined; apiKey?: string | undefined; model?: string | undefined; + providerApiKeys?: Partial> | undefined; } diff --git a/src/main.ts b/src/main.ts index 1cd0ecc..5db691e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,5 +1,5 @@ import { loadEnvironmentFromNodeEnv } from "./common/env-loader.js"; -import { getServiceName } from "./config.js"; +import { getOpenRouterApiUrl, getOpenRouterEnabled, getServiceName } from "./config.js"; import type { GarnetClient } from "./common/garnet.js"; import type { NatsClient } from "./common/nats.js"; @@ -21,6 +21,7 @@ const { NatsClient: RuntimeNatsClient } = await import("./common/nats.js"); const { Agent } = await import("./agent/index.js"); const { ConversationStore } = await import("./conversation/index.js"); const { OnboardingSkill } = await import("./skills/onboarding/index.js"); +const { ShadowEvalBus } = await import("./eval/shadow-bus.js"); const { LlmClient } = await import("./llm/index.js"); const { getRawSecret, getSecret } = await import("./common/secret-manager.js"); const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js"); @@ -53,22 +54,22 @@ function parseEndpointList( function toProviderEndpoints( urls: string[], - provider?: "default" | "cliproxy", -): Array { - if (!provider || provider === "default") { + provider?: "llama" | "cliproxy" | "openrouter", +): Array { + if (!provider) { return urls; } return urls.map((url) => ({ url, provider })); } function formatEndpointList( - endpoints: Array, + endpoints: Array, ): string { return endpoints .map((endpoint) => typeof endpoint === "string" ? endpoint - : `${endpoint.provider ?? "default"}:${endpoint.url}`, + : `${endpoint.provider}:${endpoint.url}`, ) .join(", "); } @@ -135,30 +136,50 @@ async function bootstrap(): Promise { getRawSecret("LLAMA_SERVER_API_URL"), process.env["LLAMA_SERVER_API_URL"], ); + const openRouterEndpoints = getOpenRouterEnabled() + ? parseEndpointList( + getRawSecret("OPENROUTER_API_URL"), + process.env["OPENROUTER_API_URL"], + getOpenRouterApiUrl(), + ) + : []; const llmEndpoints = [ ...toProviderEndpoints(cliProxyEndpoints, "cliproxy"), - ...toProviderEndpoints(generalLlmEndpoints), + ...toProviderEndpoints(openRouterEndpoints, "openrouter"), + ...toProviderEndpoints(generalLlmEndpoints, "llama"), ]; - const llmApiKey = firstNonEmpty( + const cliproxyApiKey = firstNonEmpty( getRawSecret("CLI_PROXY_API_KEY"), process.env["CLI_PROXY_API_KEY"], + ); + const openRouterApiKey = firstNonEmpty( + getRawSecret("OPENROUTER_API_KEY"), + process.env["OPENROUTER_API_KEY"], + ); + const llmApiKey = firstNonEmpty( getRawSecret("AGENT_LLM_API_KEY"), process.env["AGENT_LLM_API_KEY"], getRawSecret("LLAMA_SERVER_API_KEY"), process.env["LLAMA_SERVER_API_KEY"], ); const llmModel = firstNonEmpty( - getRawSecret("CLI_PROXY_LLM_MODEL"), - process.env["CLI_PROXY_LLM_MODEL"], getRawSecret("AGENT_LLM_MODEL"), process.env["AGENT_LLM_MODEL"], ); + const providerApiKeys = Object.fromEntries( + [ + ["cliproxy", cliproxyApiKey], + ["llama", llmApiKey], + ["openrouter", openRouterApiKey], + ].filter((entry): entry is [string, string] => typeof entry[1] === "string"), + ) as Partial>; const llmClient = new LlmClient({ endpoints: llmEndpoints, garnetClient, apiKey: llmApiKey, model: llmModel, + providerApiKeys, }); if (llmEndpoints.length > 0) { @@ -197,7 +218,8 @@ async function bootstrap(): Promise { const agent = new Agent(natsClient, conversationStore); // Register skills - const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService); + const shadowEvalBus = new ShadowEvalBus(garnetClient); + const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService, shadowEvalBus); agent.registerSkill(onboardingSkill); installShutdownHooks(async () => { diff --git a/src/observability/alert.test.ts b/src/observability/alert.test.ts new file mode 100644 index 0000000..520f395 --- /dev/null +++ b/src/observability/alert.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { AlertPayload } from "@common/nats-subjects"; +import { AGENT_ALERT_SUBJECT } from "@common/nats-subjects"; +import { AlertService } from "./alert.js"; + +const payload: AlertPayload = { + kind: "latency", + metric: "shadow_eval.p95_latency_ms", + severity: "warn", + threshold: 3000, + timestamp: "2026-04-17T00:00:00.000Z", + value: 4100, + window: "5m", +}; + +describe("AlertService", () => { + it("publishes when dedupe key is claimed", async () => { + const setStringIfAbsent = vi.fn().mockResolvedValue(true); + const publish = vi.fn().mockResolvedValue(undefined); + const service = new AlertService({ setStringIfAbsent }, { publish }); + + const result = await service.sendAlert(payload); + + expect(result).toBe(true); + expect(setStringIfAbsent).toHaveBeenCalledTimes(1); + expect(publish).toHaveBeenCalledWith(AGENT_ALERT_SUBJECT, payload); + }); + + it("suppresses duplicate alerts", async () => { + const setStringIfAbsent = vi.fn().mockResolvedValue(false); + const publish = vi.fn().mockResolvedValue(undefined); + const service = new AlertService({ setStringIfAbsent }, { publish }); + + const result = await service.sendAlert(payload); + + expect(result).toBe(false); + expect(publish).not.toHaveBeenCalled(); + }); +}); diff --git a/src/observability/alert.ts b/src/observability/alert.ts new file mode 100644 index 0000000..967233d --- /dev/null +++ b/src/observability/alert.ts @@ -0,0 +1,66 @@ +import type { GarnetClient } from "@common/garnet"; +import { AGENT_ALERT_SUBJECT, type AlertPayload } from "@common/nats-subjects"; +import type { NatsClient } from "@common/nats"; +import { getAlertRateLimitSec } from "../config.js"; +import { getLogger } from "./logger.js"; + +export interface AlertSender { + sendAlert( + payload: AlertPayload, + options?: { + dedupeKey?: string | undefined; + }, + ): Promise; +} + +function getAlertDedupeKey(kind: AlertPayload["kind"]): string { + return `lolympus-agent:alert:dedupe:${kind}`; +} + +export class AlertService implements AlertSender { + private readonly logger = getLogger(); + + constructor( + private readonly garnetClient: Pick, + private readonly natsClient: Pick, + ) {} + + async sendAlert( + payload: AlertPayload, + options?: { + dedupeKey?: string | undefined; + }, + ): Promise { + const dedupeKey = options?.dedupeKey ?? getAlertDedupeKey(payload.kind); + const claimed = await this.garnetClient.setStringIfAbsent( + dedupeKey, + payload.timestamp, + getAlertRateLimitSec(), + ); + + if (!claimed) { + this.logger.debug( + { + dedupeKey, + kind: payload.kind, + }, + "[Alert] Suppressed by rate limit", + ); + return false; + } + + await this.natsClient.publish(AGENT_ALERT_SUBJECT, payload); + this.logger.warn( + { + kind: payload.kind, + metric: payload.metric, + severity: payload.severity, + threshold: payload.threshold, + value: payload.value, + window: payload.window, + }, + "[Alert] Published", + ); + return true; + } +} diff --git a/src/observability/tracing.ts b/src/observability/tracing.ts index 872bbda..8c08c32 100644 --- a/src/observability/tracing.ts +++ b/src/observability/tracing.ts @@ -17,11 +17,18 @@ export function initTracing(): void { return; } + const exporterType = getOtelExporter(); + if (exporterType === 'none') { + const logger = getLogger(); + logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init'); + started = true; + return; + } + process.env['OTEL_SERVICE_NAME'] ??= getServiceName(); const logger = getLogger(); - const exporter = - getOtelExporter() === 'stdout' ? new ConsoleSpanExporter() : new ConsoleSpanExporter(); + const exporter = new ConsoleSpanExporter(); sdk = new NodeSDK({ instrumentations: [getNodeAutoInstrumentations()], diff --git a/src/skills/onboarding/constants.ts b/src/skills/onboarding/constants.ts new file mode 100644 index 0000000..3481eaf --- /dev/null +++ b/src/skills/onboarding/constants.ts @@ -0,0 +1,45 @@ +export const VALID_TIERS = [ + "GM", + "I", + "B", + "S", + "G", + "P", + "E", + "D", + "M", + "C", + "U", +] as const; + +export const VALID_TIER_SET = new Set(VALID_TIERS); +export const VALID_GENDERS = ["남", "여"] as const; +export const VALID_GENDER_SET = new Set(VALID_GENDERS); + +export const MIN_AGE = 27; +export const MAX_AGE: number | null = null; +export const NICKNAME_FORMAT_EXAMPLE = "홍길동/95/남/G" as const; +export const TIER_CODES_TEXT = VALID_TIERS.join(", "); + +export const NICKNAME_FORMAT_TEXT = + `닉네임은 이름/출생 연도 두 자리/성별/티어 형식이에요. 예를 들면 ${NICKNAME_FORMAT_EXAMPLE}처럼 쓰시면 돼요.`; + +export const HEART_RULE_TEXT = + '채팅방 오른쪽 상단 메뉴 버튼에서 "커버 보기"로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.'; + +export const HEART_LOCATION_TEXT = + '채팅방 오른쪽 상단 메뉴 버튼을 누르면 "커버 보기"가 보여요. 거기서 프로필과 하트를 확인하실 수 있어요.'; + +export const DISCORD_GUIDE_TEXT = + "닉네임 변경과 하트 인증을 모두 완료하신 뒤에 디스코드 서버에서 프로필을 설정하시면 권한이 자동으로 부여돼요."; + +export const DISCORD_INVITE_URL = + process.env["DISCORD_INVITE_URL"]?.trim() || ""; + +export const DEADLINE_NEAR_THRESHOLD_MIN = Number( + process.env["DEADLINE_NEAR_THRESHOLD_MIN"] ?? 30, +); + +export const ESCALATION_BUDGET_PER_TURN = Number( + process.env["ESCALATION_BUDGET_PER_TURN"] ?? 2, +); diff --git a/src/skills/onboarding/onboarding-templates.ts b/src/skills/onboarding/onboarding-templates.ts index 34ce804..dece511 100644 --- a/src/skills/onboarding/onboarding-templates.ts +++ b/src/skills/onboarding/onboarding-templates.ts @@ -1,12 +1,19 @@ import type { NicknameIssue, OnboardingSession } from "./types.js"; +import { + HEART_LOCATION_TEXT, + HEART_RULE_TEXT, + MIN_AGE, + NICKNAME_FORMAT_EXAMPLE, + NICKNAME_FORMAT_TEXT, + TIER_CODES_TEXT, + VALID_GENDER_SET, + VALID_TIER_SET, +} from "./constants.js"; function pickRandom(items: [string, ...string[]]): string { return items[Math.floor(Math.random() * items.length)] ?? items[0]; } -const VALID_GENDERS = new Set(["남", "여"]); -const VALID_TIERS = new Set(["GM", "I", "B", "S", "G", "P", "E", "D", "M", "C", "U"]); - export function analyzeNicknameIssues(nickname: string): NicknameIssue[] { const parts = nickname.trim().split("/"); const issues: NicknameIssue[] = []; @@ -21,9 +28,9 @@ export function analyzeNicknameIssues(nickname: string): NicknameIssue[] { }); } if (third) { - const hasGender = VALID_GENDERS.has(third.charAt(0)); + const hasGender = VALID_GENDER_SET.has(third.charAt(0)); const rest = third.slice(1); - if (hasGender && rest.length > 0 && !VALID_TIERS.has(rest)) { + if (hasGender && rest.length > 0 && !VALID_TIER_SET.has(rest)) { issues.push({ kind: "unknown-tier", message: `성별 뒤의 '${rest}'를 제거하고, 티어 코드를 '/'로 구분해서 추가해주세요 (예: ${third.charAt(0)}/D)`, @@ -74,17 +81,17 @@ export function analyzeNicknameIssues(nickname: string): NicknameIssue[] { }); } - if (!gender || !VALID_GENDERS.has(gender)) { + if (!gender || !VALID_GENDER_SET.has(gender)) { issues.push({ kind: "missing-gender", message: "성별을 '남' 또는 '여'로 입력해주세요", }); } - if (!tier || !VALID_TIERS.has(tier)) { + if (!tier || !VALID_TIER_SET.has(tier)) { issues.push({ kind: "unknown-tier", - message: "티어 코드(GM,I,B,S,G,P,E,D,M,C,U 중 하나)를 입력해주세요", + message: `티어 코드(${TIER_CODES_TEXT} 중 하나)를 입력해주세요`, }); } @@ -146,29 +153,29 @@ export function getNicknameInvalidMessage( export function getNicknameFormatMessage(): string { return pickRandom([ - "닉네임 형식은 이름/출생 연도 두 자리/성별/티어예요. 예를 들면 홍길동/95/여/D처럼 쓰시면 돼요.", - "닉네임은 이름/출생 연도 두 자리/성별/티어 형식으로 입력해 주시면 돼요! 예: 홍길동/95/남/G", + NICKNAME_FORMAT_TEXT, + `닉네임은 이름/출생 연도 두 자리/성별/티어 형식으로 입력해 주시면 돼요! 예: ${NICKNAME_FORMAT_EXAMPLE}`, ]); } export function getHeartGuideMessage(): string { return pickRandom([ - '채팅방 오른쪽 상단 메뉴 버튼에서 "커버 보기"로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.', + HEART_RULE_TEXT, "채팅방 프로필에서 하트를 눌러 빨간색으로 만든 다음, 그 화면이 보이게 원본 스크린샷을 보내주시면 인증돼요.", ]); } export function getHeartLocationMessage(): string { return pickRandom([ - '채팅방 오른쪽 상단 메뉴 버튼을 누르면 "커버 보기"가 보여요. 거기서 프로필과 하트를 확인하실 수 있어요.', + HEART_LOCATION_TEXT, '오른쪽 상단 메뉴 버튼으로 들어가 "커버 보기"를 열면 하트 위치를 확인하실 수 있어요.', ]); } export function getAgeRequirementMessage(): string { return pickRandom([ - "롤림푸스는 27세 이상만 가입이 가능해요.", - "가입 가능 연령은 27세 이상이에요.", + `롤림푸스는 ${MIN_AGE}세 이상만 가입이 가능해요.`, + `가입 가능 연령은 ${MIN_AGE}세 이상이에요.`, ]); } diff --git a/src/skills/onboarding/prompts/builder.ts b/src/skills/onboarding/prompts/builder.ts index 56f66ef..7020589 100644 --- a/src/skills/onboarding/prompts/builder.ts +++ b/src/skills/onboarding/prompts/builder.ts @@ -8,6 +8,7 @@ import { policyLayer, stateLayer, } from "./layers.js"; +import { buildToolPolicy } from "./tool-policy.js"; export class PromptBuilder { private constructor( @@ -35,7 +36,12 @@ export class PromptBuilder { const systemPrompt = [ personaLayer(), - policyLayer(this.derived.intent), + policyLayer(this.derived.intent, { + toolPolicy: + this.derived.responseMode === "llm-with-tools" + ? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind) + : undefined, + }), stateLayer(this.session, this.derived), examplesLayer(this.derived.intent), contextLayer(promptContext), diff --git a/src/skills/onboarding/prompts/layers.ts b/src/skills/onboarding/prompts/layers.ts index 25b8436..23f896a 100644 --- a/src/skills/onboarding/prompts/layers.ts +++ b/src/skills/onboarding/prompts/layers.ts @@ -7,6 +7,12 @@ import type { OnboardingSession, PromptContext, } from "../types.js"; +import { + HEART_RULE_TEXT, + MIN_AGE, + NICKNAME_FORMAT_EXAMPLE, + TIER_CODES_TEXT, +} from "../constants.js"; import { EXAMPLES_BY_INTENT } from "./examples.js"; function formatKstTimestamp(value: string | undefined): string { @@ -94,11 +100,11 @@ export function personaLayer(): string { 처음 온 멤버가 편안하게 등록을 마칠 수 있도록 자연스럽고 정확하게 안내해줘. [커뮤니티 규칙] -- 가입 가능 연령은 27세 이상이야. +- 가입 가능 연령은 ${MIN_AGE}세 이상이야. - 새내기 등록은 닉네임 변경과 방 하트 인증 두 가지를 모두 완료하면 돼. -- 닉네임 형식은 이름/출생 연도 두 자리/성별/티어야. 예: 홍길동/95/남/G -- 성별은 남 또는 여, 티어 코드는 GM, I, B, S, G, P, E, D, M, C, U 중 하나야. -- 방 하트 인증은 채팅방 프로필에서 하트를 빨간색으로 바꾼 뒤 편집 없는 원본 스크린샷을 보내는 절차야. +- 닉네임 형식은 이름/출생 연도 두 자리/성별/티어야. 예: ${NICKNAME_FORMAT_EXAMPLE} +- 성별은 남 또는 여, 티어 코드는 ${TIER_CODES_TEXT} 중 하나야. +- 방 하트 인증은 ${HEART_RULE_TEXT} - 디스코드 안내는 닉네임 변경과 하트 인증이 모두 완료된 뒤에만 해줘.`; } diff --git a/src/skills/onboarding/prompts/tool-policy.ts b/src/skills/onboarding/prompts/tool-policy.ts new file mode 100644 index 0000000..442289d --- /dev/null +++ b/src/skills/onboarding/prompts/tool-policy.ts @@ -0,0 +1,28 @@ +import type { Intent, OnboardingPhase } from "../state/onboarding-state.js"; + +export function buildToolPolicy( + intent: Intent["kind"], + phase: OnboardingPhase["kind"], +): string { + if (intent === "nickname-question" && phase === "nickname-rejected") { + return "- 닉네임 후보나 현재 닉네임을 설명할 때는 validate-nickname 도구를 먼저 사용한 뒤, 틀린 부분만 짚어."; + } + + if (intent === "status-question") { + return "- 진행 상태 질문은 check-onboarding-progress 도구를 우선 사용하고, 상태만 짧게 답해."; + } + + if (intent === "age-question" || intent === "discord-question") { + return "- 규칙 확인이 필요하면 lookup-community-rule 도구를 사용해 정확한 규칙만 답해."; + } + + if (intent === "help-where") { + return "- 위치 안내가 모호할 때만 get-room-context 도구를 고려하고, 기본적으로는 절차 위치만 답해."; + } + + if (phase === "deadline-near") { + return "- 마감이 가까운 상태에서는 남은 절차만 우선순위로 짧게 답해."; + } + + return "- 도구가 꼭 필요하지 않다면 호출하지 말고, 지금 질문에 대한 답만 간결하게 해."; +} diff --git a/src/skills/onboarding/runtime/critic.ts b/src/skills/onboarding/runtime/critic.ts new file mode 100644 index 0000000..8b16dbe --- /dev/null +++ b/src/skills/onboarding/runtime/critic.ts @@ -0,0 +1,83 @@ +import type { ChatMessage } from "@llm/types"; +import type { LlmClient } from "@llm/llm-client"; +import { + getCriticMaxTokens, + getLlmTemperature, + getTier2CriticModel, +} from "../../../config.js"; +import type { DerivedState } from "../state/onboarding-state.js"; + +export interface CritiqueResult { + changed: boolean; + text: string; +} + +function serializeMessages(messages: ChatMessage[]): string { + return messages + .map((message) => `[${message.role}] ${message.content}`) + .join("\n\n"); +} + +function buildCriticMessages( + draft: string, + derived: DerivedState, + messages: ChatMessage[], +): ChatMessage[] { + return [ + { + role: "system", + content: [ + "당신은 온보딩 응답을 검토하는 critic입니다.", + "초안이 충분히 정확하고 범위를 벗어나지 않으며 간결하면 반드시 [OK]만 출력하세요.", + "고쳐야 한다면 유저에게 바로 보낼 최종 한국어 답변만 출력하고, 설명이나 해설은 쓰지 마세요.", + "질문에서 묻지 않은 추가 절차를 길게 덧붙이지 말고, 꼭 필요한 내용만 남기세요.", + ].join("\n"), + }, + { + role: "user", + content: [ + `[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}`, + "[CONVERSATION]", + serializeMessages(messages), + "[DRAFT]", + draft, + ].join("\n\n"), + }, + ]; +} + +export class OnboardingCritic { + constructor(private readonly llmClient: LlmClient) {} + + async critique( + draft: string, + derived: DerivedState, + messages: ChatMessage[], + options?: { deadlineMs?: number | undefined }, + ): Promise { + const response = await this.llmClient.chatCompletion( + buildCriticMessages(draft, derived, messages), + { + deadlineMs: options?.deadlineMs, + maxTokens: getCriticMaxTokens(), + modelOverride: getTier2CriticModel(), + role: "critic", + temperature: Math.max(0, getLlmTemperature() - 0.15), + tier: 2, + }, + ); + + const text = response?.content.trim() ?? ""; + if (!text || text.startsWith("[OK]")) { + return { + changed: false, + text: draft, + }; + } + + return { + changed: true, + text, + }; + } +} diff --git a/src/skills/onboarding/runtime/orchestrator.test.ts b/src/skills/onboarding/runtime/orchestrator.test.ts new file mode 100644 index 0000000..b5d97a8 --- /dev/null +++ b/src/skills/onboarding/runtime/orchestrator.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, it } from "vitest"; +import type { ChatCompletionOptions, ChatMessage } from "@llm/types"; +import { OnboardingOrchestrator } from "./orchestrator.js"; +import type { DerivedState } from "../state/onboarding-state.js"; + +function createDerivedState(overrides?: Partial): DerivedState { + return { + phase: { kind: "photo-pending" }, + intent: { kind: "help-how" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 256, + allowShortResponse: false, + tier: 0, + ...overrides, + }; +} + +describe("OnboardingOrchestrator", () => { + it("uses direct response path for tier 0", async () => { + const calls: Array<{ + messages: ChatMessage[]; + options: ChatCompletionOptions | undefined; + }> = []; + const llmClientStub = { + chatCompletion: ( + messages: ChatMessage[], + options?: ChatCompletionOptions, + ) => { + calls.push({ messages, options }); + return Promise.resolve({ + content: "안내 도와드릴게요.", + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "안내 도와드릴게요." } }] }, + message: { role: "assistant" as const, content: "안내 도와드릴게요." }, + }); + }, + }; + + const orchestrator = new OnboardingOrchestrator(llmClientStub as never); + const result = await orchestrator.run( + createDerivedState({ tier: 0 }), + [{ role: "user", content: "도와줘" }], + { toolContext: {} }, + ); + + expect(result).toBe("안내 도와드릴게요."); + expect(calls).toHaveLength(1); + expect(calls[0]?.options?.role).toBe("responder"); + }); + + it("runs planner then responder for tier 1", async () => { + const calls: Array<{ + messages: ChatMessage[]; + options: ChatCompletionOptions | undefined; + }> = []; + const llmClientStub = { + chatCompletion: ( + messages: ChatMessage[], + options?: ChatCompletionOptions, + ) => { + calls.push({ messages, options }); + + if (calls.length === 1) { + return Promise.resolve({ + content: JSON.stringify({ + approach: "redirect-to-step", + targetPhase: "photo-pending", + mustInclude: ["하트 인증만 안내"], + mustAvoid: ["디스코드 절차"], + suggestedTools: [], + tone: "friendly", + targetLengthChars: 120, + rationale: "질문 범위만 답하기", + }), + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "{}" } }] }, + message: { role: "assistant" as const, content: "{}" }, + }); + } + + return Promise.resolve({ + content: "하트 인증만 남았어요.", + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] }, + message: { role: "assistant" as const, content: "하트 인증만 남았어요." }, + }); + }, + }; + + const orchestrator = new OnboardingOrchestrator(llmClientStub as never); + const result = await orchestrator.run( + createDerivedState({ tier: 1 }), + [ + { role: "system", content: "기본 시스템 프롬프트" }, + { role: "user", content: "이제 뭐 남았지" }, + ], + { toolContext: {} }, + ); + + expect(result).toBe("하트 인증만 남았어요."); + expect(calls).toHaveLength(2); + expect(calls[0]?.options?.role).toBe("planner"); + expect(calls[1]?.messages[0]?.content).toContain("[PLAN]"); + }); + + it("runs critic after responder for tier 2 and applies rewrite", async () => { + const calls: Array<{ + messages: ChatMessage[]; + options: ChatCompletionOptions | undefined; + }> = []; + const llmClientStub = { + chatCompletion: ( + messages: ChatMessage[], + options?: ChatCompletionOptions, + ) => { + calls.push({ messages, options }); + + if (calls.length === 1) { + return Promise.resolve({ + content: JSON.stringify({ + approach: "answer-direct", + targetPhase: "photo-pending", + mustInclude: [], + mustAvoid: ["추가 절차 나열"], + suggestedTools: [], + tone: "concise", + targetLengthChars: 60, + rationale: "짧게 답하기", + }), + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "{}" } }] }, + message: { role: "assistant" as const, content: "{}" }, + }); + } + + if (calls.length === 2) { + return Promise.resolve({ + content: "하트 인증만 남았어요. 커버 보기로 들어가서...", + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { + choices: [ + { message: { content: "하트 인증만 남았어요. 커버 보기로 들어가서..." } }, + ], + }, + message: { + role: "assistant" as const, + content: "하트 인증만 남았어요. 커버 보기로 들어가서...", + }, + }); + } + + return Promise.resolve({ + content: "하트 인증만 남았어요.", + endpoint: "mock://llm", + provider: "cliproxy" as const, + raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] }, + message: { role: "assistant" as const, content: "하트 인증만 남았어요." }, + }); + }, + }; + + const orchestrator = new OnboardingOrchestrator(llmClientStub as never); + const result = await orchestrator.run( + createDerivedState({ tier: 2 }), + [ + { role: "system", content: "기본 시스템 프롬프트" }, + { role: "user", content: "이제 뭐 남았지" }, + ], + { toolContext: {} }, + ); + + expect(result).toBe("하트 인증만 남았어요."); + expect(calls).toHaveLength(3); + expect(calls[2]?.options?.role).toBe("critic"); + }); +}); diff --git a/src/skills/onboarding/runtime/orchestrator.ts b/src/skills/onboarding/runtime/orchestrator.ts new file mode 100644 index 0000000..a9e7a36 --- /dev/null +++ b/src/skills/onboarding/runtime/orchestrator.ts @@ -0,0 +1,382 @@ +import { getContext } from "@obs/context"; +import type { LlmClient } from "@llm/llm-client"; +import type { ChatMessage } from "@llm/types"; +import type { Tool, ToolContext } from "@tools/tool"; +import { + getLlmTemperature, + getPlannerMaxTokens, + getTier0EstimatedCallMs, + getTier0MaxTokens, + getTier0Model, + getTier1EstimatedCallMs, + getTier1MaxTokens, + getTier1PlannerModel, + getTier1ResponderModel, + getTier2EstimatedCallMs, + getTier2MaxTokens, +} from "../../../config.js"; +import type { DerivedState } from "../state/onboarding-state.js"; +import { OnboardingCritic } from "./critic.js"; +import type { PlannerOutput } from "./planner-schema.js"; +import { PlannerOutputSchema } from "./planner-schema.js"; +import { runWithTools } from "./tool-loop.js"; + +export interface OrchestratorRunOptions { + maxTokens?: number | undefined; + temperature?: number | undefined; + toolContext: ToolContext; + tools?: readonly Tool[] | undefined; +} + +function cloneMessages(messages: ChatMessage[]): ChatMessage[] { + return messages.map((message) => ({ + ...message, + tool_calls: message.tool_calls ? [...message.tool_calls] : undefined, + })); +} + +function injectSystemAddendum( + messages: ChatMessage[], + addendum: string, +): ChatMessage[] { + const cloned = cloneMessages(messages); + const firstSystemIndex = cloned.findIndex((message) => message.role === "system"); + + if (firstSystemIndex === -1) { + cloned.unshift({ + role: "system", + content: addendum, + }); + return cloned; + } + + const current = cloned[firstSystemIndex]; + if (!current) { + return cloned; + } + + cloned[firstSystemIndex] = { + ...current, + content: `${current.content}\n\n${addendum}`, + }; + return cloned; +} + +function summarizeMessages(messages: ChatMessage[]): string { + return messages + .map((message) => `[${message.role}] ${message.content}`) + .join("\n\n"); +} + +function tryParsePlannerOutput(raw: string): PlannerOutput | null { + try { + const parsed = JSON.parse(raw) as unknown; + return PlannerOutputSchema.parse(parsed); + } catch { + const start = raw.indexOf("{"); + const end = raw.lastIndexOf("}"); + if (start === -1 || end === -1 || end <= start) { + return null; + } + + try { + const parsed = JSON.parse(raw.slice(start, end + 1)) as unknown; + return PlannerOutputSchema.parse(parsed); + } catch { + return null; + } + } +} + +function getDefaultPlan(derived: DerivedState): PlannerOutput { + return { + approach: + derived.responseMode === "llm-with-tools" + ? "validate-then-explain" + : derived.intent.kind === "help-how" || derived.intent.kind === "help-where" + ? "redirect-to-step" + : "answer-direct", + targetPhase: derived.phase.kind, + mustInclude: [], + mustAvoid: [], + suggestedTools: + derived.responseMode === "llm-with-tools" ? ["validate-nickname"] : [], + tone: derived.allowShortResponse ? "concise" : "friendly", + targetLengthChars: derived.allowShortResponse ? 80 : 150, + rationale: "fallback-plan", + }; +} + +function getRemainingDeadlineMs(): number | undefined { + const deadlineAt = getContext()?.deadlineAt; + if (!deadlineAt) { + return undefined; + } + + const deadlineMs = Date.parse(deadlineAt) - Date.now(); + return Number.isFinite(deadlineMs) ? deadlineMs : undefined; +} + +function canAffordCall(estimatedCallMs: number): boolean { + const remainingMs = getRemainingDeadlineMs(); + return remainingMs === undefined || remainingMs > estimatedCallMs; +} + +function buildPlannerMessages( + derived: DerivedState, + messages: ChatMessage[], +): ChatMessage[] { + return [ + { + role: "system", + content: [ + "당신은 온보딩 응답을 설계하는 planner입니다.", + "반드시 JSON만 출력하세요.", + "질문에 직접 답할 수 있으면 answer-direct를 우선하고, 유저가 묻지 않은 추가 절차는 mustAvoid에 넣으세요.", + "tool은 꼭 필요한 경우에만 suggestedTools에 넣고 최대 3개까지만 고르세요.", + ].join("\n"), + }, + { + role: "user", + content: [ + `[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}, tier=${derived.tier}`, + "[CONVERSATION]", + summarizeMessages(messages), + "[SCHEMA]", + JSON.stringify({ + approach: + "answer-direct | ask-clarification | redirect-to-step | validate-then-explain | lookup-rule-then-answer | soft-decline", + targetPhase: + "fresh | nickname-pending | photo-pending | both-pending | nickname-rejected | photo-rejected | deadline-near | completed | timed-out", + mustInclude: ["short strings"], + mustAvoid: ["short strings"], + suggestedTools: [ + "validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context", + ], + tone: "friendly | concise | encouraging | neutral | apologetic", + targetLengthChars: 150, + rationale: "short reason", + }), + ].join("\n\n"), + }, + ]; +} + +function buildResponderAddendum(plan: PlannerOutput): string { + const lines = [ + "[PLAN]", + `approach=${plan.approach}`, + `tone=${plan.tone}`, + `targetLengthChars=${plan.targetLengthChars}`, + ]; + + if (plan.targetPhase) { + lines.push(`targetPhase=${plan.targetPhase}`); + } + + lines.push( + `mustInclude=${plan.mustInclude.length > 0 ? plan.mustInclude.join(" | ") : "-"}`, + `mustAvoid=${plan.mustAvoid.length > 0 ? plan.mustAvoid.join(" | ") : "-"}`, + "위 계획을 참고하되, 실제 유저에게 보낼 최종 한국어 답변만 출력하세요.", + ); + + return lines.join("\n"); +} + +function pickToolsForPlan( + plan: PlannerOutput, + tools: readonly Tool[], +): readonly Tool[] { + if (plan.approach === "answer-direct" || plan.approach === "soft-decline") { + return []; + } + + if (plan.suggestedTools.length === 0) { + return tools; + } + + const picked = tools.filter((tool) => plan.suggestedTools.includes(tool.name as never)); + return picked.length > 0 ? picked : tools; +} + +export class OnboardingOrchestrator { + private readonly critic: OnboardingCritic; + + constructor(private readonly llmClient: LlmClient) { + this.critic = new OnboardingCritic(llmClient); + } + + private async runDirect( + derived: DerivedState, + messages: ChatMessage[], + options: OrchestratorRunOptions, + responderTools: readonly Tool[], + maxTokens: number, + modelOverride: string, + ): Promise { + const deadlineMs = getRemainingDeadlineMs(); + + if (responderTools.length > 0) { + return runWithTools({ + deadlineMs, + llmClient: this.llmClient, + maxTokens, + messages, + modelOverride, + role: "responder", + tier: derived.tier, + temperature: options.temperature ?? getLlmTemperature(), + toolContext: options.toolContext, + tools: responderTools, + }); + } + + const response = await this.llmClient.chatCompletion(messages, { + deadlineMs, + maxTokens, + modelOverride, + role: "responder", + temperature: options.temperature ?? getLlmTemperature(), + tier: derived.tier, + }); + + return response?.content ?? null; + } + + private async runPlanner( + derived: DerivedState, + messages: ChatMessage[], + ): Promise { + const plannerMessages = buildPlannerMessages(derived, messages); + const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, { + deadlineMs: getRemainingDeadlineMs(), + jsonMode: true, + maxTokens: getPlannerMaxTokens(), + modelOverride: getTier1PlannerModel(), + role: "planner", + temperature: 0, + tier: 1, + }); + + const firstPlan = firstAttempt?.content + ? tryParsePlannerOutput(firstAttempt.content) + : null; + if (firstPlan) { + return firstPlan; + } + + const retryMessages = [ + ...plannerMessages, + { + role: "user" as const, + content: + "방금 응답은 스키마를 벗어났어요. 설명 없이 JSON object 하나만 다시 출력하세요.", + }, + ]; + + const retry = await this.llmClient.chatCompletion(retryMessages, { + deadlineMs: getRemainingDeadlineMs(), + jsonMode: true, + maxTokens: getPlannerMaxTokens(), + modelOverride: getTier1PlannerModel(), + role: "planner", + temperature: 0, + tier: 1, + }); + + const retryPlan = retry?.content ? tryParsePlannerOutput(retry.content) : null; + return retryPlan ?? getDefaultPlan(derived); + } + + private async runTier0( + derived: DerivedState, + messages: ChatMessage[], + options: OrchestratorRunOptions, + ): Promise { + const maxTokens = options.maxTokens ?? getTier0MaxTokens(); + const tools = + derived.responseMode === "llm-with-tools" ? (options.tools ?? []) : []; + + return this.runDirect( + derived, + messages, + options, + tools, + maxTokens, + getTier0Model(), + ); + } + + private async runTier1( + derived: DerivedState, + messages: ChatMessage[], + options: OrchestratorRunOptions, + ): Promise { + const plan = await this.runPlanner(derived, messages); + const responderMessages = injectSystemAddendum( + messages, + buildResponderAddendum(plan), + ); + const responderTools = pickToolsForPlan(plan, options.tools ?? []); + + return this.runDirect( + derived, + responderMessages, + options, + responderTools, + options.maxTokens ?? getTier1MaxTokens(), + getTier1ResponderModel(), + ); + } + + private async runTier2( + derived: DerivedState, + messages: ChatMessage[], + options: OrchestratorRunOptions, + ): Promise { + const plan = await this.runPlanner(derived, messages); + const responderMessages = injectSystemAddendum( + messages, + buildResponderAddendum(plan), + ); + const responderTools = pickToolsForPlan(plan, options.tools ?? []); + + const draft = await this.runDirect( + derived, + responderMessages, + options, + responderTools, + options.maxTokens ?? getTier2MaxTokens(), + getTier1ResponderModel(), + ); + + if (!draft) { + return null; + } + + const critique = await this.critic.critique(draft, derived, responderMessages, { + deadlineMs: getRemainingDeadlineMs(), + }); + return critique.text; + } + + async run( + derived: DerivedState, + messages: ChatMessage[], + options: OrchestratorRunOptions, + ): Promise { + if (derived.tier >= 2 && canAffordCall(getTier2EstimatedCallMs())) { + return this.runTier2(derived, messages, options); + } + + if (derived.tier >= 1 && canAffordCall(getTier1EstimatedCallMs())) { + return this.runTier1(derived, messages, options); + } + + if (!canAffordCall(getTier0EstimatedCallMs())) { + return null; + } + + return this.runTier0(derived, messages, options); + } +} diff --git a/src/skills/onboarding/runtime/planner-schema.ts b/src/skills/onboarding/runtime/planner-schema.ts new file mode 100644 index 0000000..8718d02 --- /dev/null +++ b/src/skills/onboarding/runtime/planner-schema.ts @@ -0,0 +1,50 @@ +import { z } from "zod"; + +export const PlannerApproachSchema = z.enum([ + "answer-direct", + "ask-clarification", + "redirect-to-step", + "validate-then-explain", + "lookup-rule-then-answer", + "soft-decline", +]); + +export const PlannerTargetPhaseSchema = z.enum([ + "fresh", + "nickname-pending", + "photo-pending", + "both-pending", + "nickname-rejected", + "photo-rejected", + "deadline-near", + "completed", + "timed-out", +]); + +export const PlannerSuggestedToolSchema = z.enum([ + "validate-nickname", + "check-onboarding-progress", + "lookup-community-rule", + "get-room-context", +]); + +export const PlannerToneSchema = z.enum([ + "friendly", + "concise", + "encouraging", + "neutral", + "apologetic", +]); + +export const PlannerOutputSchema = z.object({ + approach: PlannerApproachSchema, + targetPhase: PlannerTargetPhaseSchema.optional(), + mustInclude: z.array(z.string().min(1).max(80)).max(5).default([]), + mustAvoid: z.array(z.string().min(1).max(80)).max(5).default([]), + suggestedTools: z.array(PlannerSuggestedToolSchema).max(3).default([]), + tone: PlannerToneSchema.default("friendly"), + targetLengthChars: z.number().int().min(20).max(500).default(150), + rationale: z.string().max(200), +}); + +export type PlannerOutput = z.infer; diff --git a/src/skills/onboarding/runtime/tool-loop.test.ts b/src/skills/onboarding/runtime/tool-loop.test.ts new file mode 100644 index 0000000..1317dca --- /dev/null +++ b/src/skills/onboarding/runtime/tool-loop.test.ts @@ -0,0 +1,219 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { runWithTools } from "./tool-loop.js"; +import type { Tool } from "@tools/tool"; + +describe("runWithTools", () => { + it("executes a tool call and returns the model's final answer", async () => { + const calls: Array<{ + content: string; + messages: Array<{ role: string; content: string }>; + options?: Record | undefined; + }> = []; + + const llmClientStub = { + chatCompletion: ( + messages: Array<{ role: string; content: string }>, + options?: Record, + ) => { + calls.push({ + content: messages[messages.length - 1]?.content ?? "", + messages, + options, + }); + + if (calls.length === 1) { + return Promise.resolve({ + content: + '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', + endpoint: "mock://llm", + provider: "llama", + raw: { + choices: [ + { + message: { + content: + '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', + }, + }, + ], + }, + message: { + role: "assistant", + content: + '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}', + }, + }); + } + + return Promise.resolve({ + content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D", + endpoint: "mock://llm", + provider: "llama", + raw: { + choices: [ + { + message: { + content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D", + }, + }, + ], + }, + message: { + role: "assistant", + content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D", + }, + }); + }, + }; + + const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = { + name: "validate-nickname", + description: "닉네임 형식 검사", + argsSchema: z.object({ + nickname: z.string(), + }), + resultSchema: z.object({ + valid: z.boolean(), + issues: z.array(z.string()), + }), + execute: () => + Promise.resolve({ + valid: false, + issues: ["티어는 '/'로 구분해서 넣어주세요."], + }), + }; + + const result = await runWithTools({ + llmClient: llmClientStub as never, + messages: [ + { + role: "system", + content: "도구를 필요할 때만 사용해.", + }, + { + role: "user", + content: "닉네임이 왜 안 돼요?", + }, + ], + toolContext: {}, + tools: [tool], + }); + + expect(result).toContain("프리리/95/여/D"); + expect(calls[0]?.options?.tools).toHaveLength(1); + expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true); + }); + + it("preserves native tool call responses for follow-up turns", async () => { + const seenMessages: Array>> = []; + + const llmClientStub = { + chatCompletion: (messages: Array>) => { + seenMessages.push(messages); + + if (seenMessages.length === 1) { + return Promise.resolve({ + content: "", + endpoint: "mock://cliproxy", + provider: "cliproxy", + raw: { + choices: [ + { + message: { + content: null, + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "lookup-community-rule", + arguments: '{"topic":"age"}', + }, + }, + ], + }, + }, + ], + }, + message: { + role: "assistant", + content: "", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "lookup-community-rule", + arguments: '{"topic":"age"}', + }, + }, + ], + }, + }); + } + + return Promise.resolve({ + content: "나이 기준은 27세 이상이에요.", + endpoint: "mock://cliproxy", + provider: "cliproxy", + raw: { + choices: [ + { + message: { + content: "나이 기준은 27세 이상이에요.", + }, + }, + ], + }, + message: { + role: "assistant", + content: "나이 기준은 27세 이상이에요.", + }, + }); + }, + }; + + const tool: Tool<{ topic: string }, { topic: string; text: string }> = { + name: "lookup-community-rule", + description: "커뮤니티 규칙 조회", + argsSchema: z.object({ + topic: z.string(), + }), + resultSchema: z.object({ + topic: z.string(), + text: z.string(), + }), + execute: () => + Promise.resolve({ + topic: "age", + text: "나이 기준은 27세 이상이에요.", + }), + }; + + const result = await runWithTools({ + llmClient: llmClientStub as never, + messages: [ + { + role: "system", + content: "필요할 때만 도구를 사용해.", + }, + { + role: "user", + content: "나이 기준이 뭐예요?", + }, + ], + toolContext: {}, + tools: [tool], + }); + + expect(result).toBe("나이 기준은 27세 이상이에요."); + expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true); + expect( + seenMessages[1]?.some( + (message) => Array.isArray(message.tool_calls) && message.role === "assistant", + ), + ).toBe(true); + }); +}); diff --git a/src/skills/onboarding/runtime/tool-loop.ts b/src/skills/onboarding/runtime/tool-loop.ts new file mode 100644 index 0000000..7e6b414 --- /dev/null +++ b/src/skills/onboarding/runtime/tool-loop.ts @@ -0,0 +1,89 @@ +import { withToolSpan } from "@obs/spans"; +import type { LlmClient } from "@llm/llm-client"; +import type { ChatCompletionOptions, ChatMessage } from "@llm/types"; +import { parseToolCalls } from "@tools/parser"; +import type { Tool, ToolContext } from "@tools/tool"; + +export interface ToolLoopOptions { + deadlineMs?: number | undefined; + llmClient: LlmClient; + maxHops?: number | undefined; + maxTokens?: number | undefined; + messages: ChatMessage[]; + modelOverride?: string | undefined; + role?: ChatCompletionOptions["role"]; + signal?: AbortSignal | undefined; + temperature?: number | undefined; + tier?: ChatCompletionOptions["tier"]; + toolContext: ToolContext; + tools: readonly Tool[]; +} + +export async function runWithTools( + options: ToolLoopOptions, +): Promise { + const maxHops = options.maxHops ?? 2; + const messages = options.messages.map((message) => ({ ...message })); + let allowNativeTools = true; + + for (let hop = 0; hop <= maxHops; hop++) { + const response = await options.llmClient.chatCompletion(messages, { + deadlineMs: options.deadlineMs, + maxTokens: options.maxTokens, + modelOverride: options.modelOverride, + role: options.role, + signal: options.signal, + temperature: options.temperature, + tier: options.tier, + tools: allowNativeTools ? options.tools : undefined, + }); + + if (!response) { + return null; + } + + const toolCalls = parseToolCalls(response.message); + if (toolCalls.length === 0) { + return response.content; + } + + if (hop === maxHops) { + messages.push(response.message); + messages.push({ + role: "user", + content: + "도구 호출은 여기까지 하고, 지금까지 얻은 정보만으로 한국어 문장으로 최종 답변해줘.", + }); + allowNativeTools = false; + continue; + } + + messages.push(response.message); + + for (const call of toolCalls) { + const tool = options.tools.find((candidate) => candidate.name === call.name); + if (!tool) { + throw new Error(`Tool not registered for call: ${call.name}`); + } + + const validated = tool.argsSchema.parse(call.arguments); + const result = await withToolSpan( + tool.name, + { + "tool.call.id": call.id, + "tool.name": tool.name, + }, + () => tool.execute(validated, options.toolContext), + ); + + messages.push({ + role: "tool", + name: tool.name, + tool_call_id: call.id, + content: JSON.stringify(result), + }); + } + } + + return null; +} diff --git a/src/skills/onboarding/runtime/validator.test.ts b/src/skills/onboarding/runtime/validator.test.ts new file mode 100644 index 0000000..4a84278 --- /dev/null +++ b/src/skills/onboarding/runtime/validator.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; + +import { validateOnboardingResponse } from "./validator.js"; +import type { DerivedState } from "../state/onboarding-state.js"; + +function makeDerived( + overrides?: Partial, +): DerivedState { + return { + phase: { kind: "photo-pending" }, + intent: { kind: "photo-question" }, + shouldRespond: true, + responseMode: "llm", + suggestedMaxTokens: 256, + allowShortResponse: false, + tier: 0, + ...overrides, + }; +} + +describe("validateOnboardingResponse", () => { + it("rejects discord guidance before onboarding is completed", () => { + const result = validateOnboardingResponse( + "디스코드 서버에 들어가시면 돼요.", + makeDerived(), + ); + + expect(result.status).toBe("reject"); + expect(result.reasons).toContain("stateConsistency"); + }); + + it("rejects invalid nickname examples", () => { + const result = validateOnboardingResponse( + "홍길동/95/남/Z로 바꾸시면 돼요.", + makeDerived({ + phase: { kind: "nickname-rejected", current: "홍길동/95/남/골드", issues: [] }, + intent: { kind: "nickname-question" }, + }), + ); + + expect(result.status).toBe("reject"); + expect(result.reasons).toContain("nicknameExampleConsistency"); + }); + + it("soft-warns on repeated phrases", () => { + const result = validateOnboardingResponse( + "하트 인증 해주세요 하트 인증 해주세요 하트 인증 해주세요", + makeDerived(), + { allowShortResponse: true }, + ); + + expect(result.status).toBe("soft-warn"); + expect(result.reasons).toContain("phraseRepetition"); + }); + + it("passes short status responses when allowShortResponse is enabled", () => { + const result = validateOnboardingResponse( + "이제 방 하트 인증만 남았어요.", + makeDerived({ + intent: { kind: "status-question" }, + }), + { allowShortResponse: true }, + ); + + expect(result.status).toBe("pass"); + }); +}); diff --git a/src/skills/onboarding/runtime/validator.ts b/src/skills/onboarding/runtime/validator.ts new file mode 100644 index 0000000..19f86cc --- /dev/null +++ b/src/skills/onboarding/runtime/validator.ts @@ -0,0 +1,180 @@ +import { trace } from "@opentelemetry/api"; +import { getMaxResponseLength } from "../../../config.js"; +import { VALID_TIER_SET } from "../constants.js"; +import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js"; +import type { DerivedState } from "../state/onboarding-state.js"; + +const ALLOWED_URL_PATTERNS = [ + /discord\.gg\/lolympus/iu, + /open\.kakao\.com\/o\/gOscBAEh/iu, +]; + +export interface ValidationResult { + status: "pass" | "soft-warn" | "reject"; + text: string; + reasons: string[]; +} + +function postProcessResponse(text: string): string { + let processed = text.trim(); + + processed = processed.replace(/[\s\S]*?<\/think>/gu, "").trim(); + + processed = processed.replace( + /https?:\/\/[^\s)}\]]+/gu, + (match) => { + if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) { + return match; + } + return ""; + }, + ); + + processed = processed.replace(/ +/gu, " ").trim(); + processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1"); + processed = processed.replace(/\*(.+?)\*/gu, "$1"); + processed = processed.replace(/__(.+?)__/gu, "$1"); + processed = processed.replace(/~~(.+?)~~/gu, "$1"); + processed = processed.replace(/^#{1,6}\s+/gmu, ""); + processed = processed.replace(/^[-*]\s+/gmu, ""); + processed = processed.replace(/`(.+?)`/gu, "$1"); + + const maxLen = getMaxResponseLength(); + if (processed.length > maxLen) { + processed = processed.slice(0, maxLen).trimEnd(); + const sentenceEndPattern = + /(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u; + + for (let index = processed.length; index > maxLen * 0.5; index--) { + const sub = processed.slice(0, index); + if (sentenceEndPattern.test(sub)) { + processed = sub; + break; + } + } + } + + return processed.trim(); +} + +function findInvalidNicknameExamples(text: string): string[] { + const matches = [ + ...text.matchAll(/([^\s/]{1,20})\/(\d{2})\/(남|여)\/([A-Z]{1,2})/gu), + ]; + + return matches.flatMap((match) => { + const tier = match[4]; + if (!tier || VALID_TIER_SET.has(tier)) { + return []; + } + return [tier]; + }); +} + +function hasIntentKeywordMatch(text: string, derived: DerivedState): boolean { + const keywords = EXPECTED_KEYWORDS[derived.intent.kind]; + if (keywords.length === 0) { + return true; + } + + const lower = text.toLowerCase(); + return keywords.some((keyword) => lower.includes(keyword.toLowerCase())); +} + +function hasRepeatedPhrase(text: string): boolean { + const words = text.split(/\s+/u); + if (words.length < 6) { + return false; + } + + const phrases = new Map(); + for (let index = 0; index < words.length - 1; index++) { + const bigram = `${words[index]} ${words[index + 1]}`; + phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1); + } + + return [...phrases.values()].some((count) => count >= 3); +} + +function markValidation(result: ValidationResult): ValidationResult { + const activeSpan = trace.getActiveSpan(); + activeSpan?.setAttributes({ + "agent.validator.status": result.status, + "agent.validator.warnings": result.reasons, + }); + return result; +} + +export function validateOnboardingResponse( + draft: string, + derived: DerivedState, + options?: { allowShortResponse?: boolean | undefined }, +): ValidationResult { + const text = postProcessResponse(draft); + const rejectReasons: string[] = []; + const softWarnReasons: string[] = []; + + if (!text) { + rejectReasons.push("empty"); + } + + if (text.includes("[NO_RESPONSE]")) { + return markValidation({ + status: "pass", + text: "[NO_RESPONSE]", + reasons: [], + }); + } + + if (!options?.allowShortResponse && text.length < 10) { + rejectReasons.push("minLength"); + } + + const mentionsDiscord = /(디스코드|discord|디코)/iu.test(text); + const advancesDiscordStep = /(서버|프로필|권한|입장)/iu.test(text); + const defersDiscordStep = + /(먼저|완료.*후|완료하신 뒤|완료하신 후|모두 완료|완료되면)/u.test(text); + + if ( + mentionsDiscord && + derived.phase.kind !== "completed" && + advancesDiscordStep && + !defersDiscordStep + ) { + rejectReasons.push("stateConsistency"); + } + + if (findInvalidNicknameExamples(text).length > 0) { + rejectReasons.push("nicknameExampleConsistency"); + } + + if (!hasIntentKeywordMatch(text, derived)) { + softWarnReasons.push("intentAlignment"); + } + + if (hasRepeatedPhrase(text)) { + softWarnReasons.push("phraseRepetition"); + } + + if (rejectReasons.length > 0) { + return markValidation({ + status: "reject", + text, + reasons: [...rejectReasons, ...softWarnReasons], + }); + } + + if (softWarnReasons.length > 0) { + return markValidation({ + status: "soft-warn", + text, + reasons: softWarnReasons, + }); + } + + return markValidation({ + status: "pass", + text, + reasons: [], + }); +} diff --git a/src/skills/onboarding/skill.ts b/src/skills/onboarding/skill.ts index 82285ff..f86d848 100644 --- a/src/skills/onboarding/skill.ts +++ b/src/skills/onboarding/skill.ts @@ -1,11 +1,17 @@ import type { AgentRequest, AgentResponse, ConversationMessage } from "@agent/types"; import { getConversationSummaryKey } from "@conv/conversation-store"; import type { GarnetClient } from "@common/garnet"; -import { getRequestLogger } from "@obs/context"; +import { getContext, getRequestLogger } from "@obs/context"; import type { LlmClient } from "@llm/llm-client"; import type { ChatMessage } from "@llm/types"; import type { RoomContextService } from "@room/room-context-service"; import type { Skill } from "@skills/skill"; +import { + type ShadowEvalPublisher, + ShadowEvalBus, +} from "@eval/shadow-bus"; +import { createBuiltInToolRegistry } from "@tools/builtins/index"; +import type { Tool, ToolContext } from "@tools/tool"; import { PromptBuilder } from "./prompts/builder.js"; import { getAcknowledgementMessage, @@ -23,103 +29,40 @@ import { getRemainingStatusMessage, getTimeoutWarningMessage, } from "./onboarding-templates.js"; -import { getOnboardingSessionKeyPrefix, getMaxResponseLength, getLlmTemperature } from "../../config.js"; +import { getOnboardingSessionKeyPrefix, getLlmTemperature } from "../../config.js"; import { OnboardingBlackboardStore } from "./state/blackboard.js"; import { deriveState } from "./state/derive.js"; import { OnboardingIntentClassifier } from "./state/classifier.js"; import type { DerivedState, Intent } from "./state/onboarding-state.js"; +import { OnboardingOrchestrator } from "./runtime/orchestrator.js"; +import { validateOnboardingResponse } from "./runtime/validator.js"; import type { OnboardingSession, PromptContext } from "./types.js"; -const ALLOWED_URL_PATTERNS = [ - /discord\.gg\/lolympus/u, - /open\.kakao\.com\/o\/gOscBAEh/u, -]; - type LlmTextResult = | { status: "success"; text: string } | { status: "no_response" } | { status: "failed" }; -function postProcessResponse(text: string): string { - let processed = text.trim(); - - // Remove blocks (Qwen thinking mode artifacts) - processed = processed.replace(/[\s\S]*?<\/think>/gu, "").trim(); - - // Strip URLs not in allowlist - processed = processed.replace( - /https?:\/\/[^\s)}\]]+/gu, - (match) => { - if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) { - return match; - } - return ""; - }, - ); - - // Clean up double spaces from URL removal - processed = processed.replace(/ +/gu, " ").trim(); - - // Strip markdown formatting (KakaoTalk is plaintext) - processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1"); // **bold** - processed = processed.replace(/\*(.+?)\*/gu, "$1"); // *italic* - processed = processed.replace(/__(.+?)__/gu, "$1"); // __underline__ - processed = processed.replace(/~~(.+?)~~/gu, "$1"); // ~~strike~~ - processed = processed.replace(/^#{1,6}\s+/gmu, ""); // ## headings - processed = processed.replace(/^[-*]\s+/gmu, ""); // - or * list items - processed = processed.replace(/`(.+?)`/gu, "$1"); // `inline code` - - // Truncate to max length with Korean sentence boundary awareness - const maxLen = getMaxResponseLength(); - if (processed.length > maxLen) { - processed = processed.slice(0, maxLen).trimEnd(); - // Search backwards for a Korean sentence-ending pattern - const sentenceEndPattern = /(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u; - // Try progressively shorter substrings to find a sentence boundary - for (let i = processed.length; i > maxLen * 0.5; i--) { - const sub = processed.slice(0, i); - if (sentenceEndPattern.test(sub)) { - processed = sub; - break; - } - } - } - - return processed.trim(); -} - -function isLowQualityResponse( - text: string, - options?: { allowShortResponse?: boolean }, -): boolean { - if (!options?.allowShortResponse && text.length < 10) return true; - // Check for excessive repetition (same phrase 3+ times) - const words = text.split(/\s+/u); - if (words.length >= 6) { - const phrases = new Map(); - for (let i = 0; i < words.length - 1; i++) { - const bigram = `${words[i]} ${words[i + 1]}`; - phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1); - } - for (const count of phrases.values()) { - if (count >= 3) return true; - } - } - return false; -} - export class OnboardingSkill implements Skill { readonly name = "onboarding"; private readonly classifier: OnboardingIntentClassifier; private readonly blackboardStore: OnboardingBlackboardStore; + private readonly orchestrator: OnboardingOrchestrator; + private readonly shadowEvalPublisher: ShadowEvalPublisher; + private readonly tools: readonly Tool[]; constructor( private readonly garnetClient: GarnetClient, private readonly llmClient: LlmClient, private readonly roomContextService?: RoomContextService, + shadowEvalPublisher?: ShadowEvalPublisher, ) { this.classifier = new OnboardingIntentClassifier(llmClient); this.blackboardStore = new OnboardingBlackboardStore(garnetClient); + this.orchestrator = new OnboardingOrchestrator(llmClient); + this.shadowEvalPublisher = + shadowEvalPublisher ?? new ShadowEvalBus(garnetClient); + this.tools = createBuiltInToolRegistry().list(); } canHandle(request: AgentRequest): boolean { @@ -185,6 +128,15 @@ export class OnboardingSkill implements Skill { await this.rememberTurn(request, derived).catch((error) => { logger.warn({ err: error }, "[OnboardingSkill] Failed to update blackboard"); }); + + this.shadowEvalPublisher.publish({ + classified, + createdAt: new Date().toISOString(), + derived, + latencyMs: this.getCurrentLatencyMs(), + request, + response, + }); } return response; @@ -212,11 +164,15 @@ export class OnboardingSkill implements Skill { derived, ); - return this.generateLlmText(messages, { + return this.generateLlmText(messages, derived, { allowShortResponse: derived.allowShortResponse, maxTokens: derived.suggestedMaxTokens, retryOnLowQuality: - request.trigger === "user-message" && !derived.allowShortResponse, + request.trigger === "user-message" && + !derived.allowShortResponse && + derived.responseMode !== "llm-with-tools", + toolContext: this.buildToolContext(request, session), + tools: this.selectToolsForDerived(derived), }); } @@ -456,11 +412,14 @@ export class OnboardingSkill implements Skill { private async generateLlmText( messages: ChatMessage[], + derived: DerivedState, options?: { allowShortResponse?: boolean; maxTokens?: number; retryOnLowQuality?: boolean; temperature?: number; + toolContext?: ToolContext | undefined; + tools?: readonly Tool[] | undefined; }, ): Promise { if (!this.llmClient.hasEndpoints()) { @@ -509,39 +468,79 @@ export class OnboardingSkill implements Skill { requestOptions.temperature = attempt.temperature; } - const llmResult = await this.llmClient.chatCompletion( + const toolContext = options?.toolContext ?? {}; + const llmText = await this.orchestrator.run( + derived, messages, - requestOptions, + { + maxTokens: requestOptions.maxTokens, + temperature: requestOptions.temperature, + toolContext, + tools: options?.tools, + }, ); - if (!llmResult) { + if (!llmText) { continue; } - const processed = postProcessResponse(llmResult.content); - if (!processed) { - continue; - } - if (processed.includes("[NO_RESPONSE]")) { + const validation = validateOnboardingResponse( + llmText, + derived, + { + allowShortResponse: options?.allowShortResponse, + }, + ); + + if (validation.text.includes("[NO_RESPONSE]")) { return { status: "no_response" }; } - const qualityOptions = - options?.allowShortResponse !== undefined - ? { allowShortResponse: options.allowShortResponse } - : undefined; - if (isLowQualityResponse(processed, qualityOptions)) { + if (validation.status === "reject") { continue; } return { status: "success", - text: processed, + text: validation.text, }; } return { status: "failed" }; } + private buildToolContext( + request: AgentRequest, + session: OnboardingSession | null, + ): ToolContext { + return { + chatroomId: request.chatroomId, + userId: request.userId, + session, + garnetClient: this.garnetClient, + roomContextService: this.roomContextService, + }; + } + + private selectToolsForDerived( + derived: DerivedState, + ): readonly Tool[] { + if (derived.phase.kind === "nickname-rejected") { + return this.tools.filter((tool) => + tool.name === "validate-nickname" || + tool.name === "lookup-community-rule" || + tool.name === "check-onboarding-progress", + ); + } + + if (derived.intent.kind === "status-question") { + return this.tools.filter((tool) => + tool.name === "check-onboarding-progress", + ); + } + + return this.tools; + } + private makeTemplateResponse( requestId: string, text: string, @@ -574,4 +573,14 @@ export class OnboardingSkill implements Skill { source: "template", }; } + + private getCurrentLatencyMs(): number | undefined { + const startedAt = getContext()?.startedAt; + const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN; + if (!Number.isFinite(startedTimestamp)) { + return undefined; + } + + return Math.max(0, Date.now() - startedTimestamp); + } } diff --git a/src/skills/onboarding/state/intent-keywords.ts b/src/skills/onboarding/state/intent-keywords.ts index ed89280..fb6945a 100644 --- a/src/skills/onboarding/state/intent-keywords.ts +++ b/src/skills/onboarding/state/intent-keywords.ts @@ -1,16 +1,16 @@ import type { Intent } from "./onboarding-state.js"; export const EXPECTED_KEYWORDS: Record = { - acknowledgement: ["감사", "알겠", "확인"], - "age-question": ["27", "나이", "가입"], + acknowledgement: ["네", "좋아요", "천천히"], + "age-question": ["27", "가입", "가능"], chitchat: [], - "discord-question": ["디스코드", "디코", "discord"], - greeting: ["안녕", "반갑", "환영"], - "help-how": ["어떻게", "방법", "해"], - "help-where": ["어디", "위치", "찾"], - "nickname-question": ["닉네임", "닉변", "형식", "이름"], + "discord-question": ["디스코드", "권한", "완료"], + greeting: ["반갑", "환영", "어서 오세요"], + "help-how": ["하트", "닉네임", "스크린샷", "형식", "커버 보기"], + "help-where": ["오른쪽 상단", "커버 보기", "프로필"], + "nickname-question": ["닉네임", "형식", "티어", "/"], "off-topic-to-other": [], - "photo-question": ["하트", "인증", "스크린샷", "프로필"], - "status-question": ["남았", "완료", "진행", "상태"], + "photo-question": ["하트", "스크린샷", "프로필", "커버 보기"], + "status-question": ["남았", "완료", "아직", "끝"], unknown: [], }; diff --git a/src/skills/onboarding/state/transitions.ts b/src/skills/onboarding/state/transitions.ts index 19cafe2..0566870 100644 --- a/src/skills/onboarding/state/transitions.ts +++ b/src/skills/onboarding/state/transitions.ts @@ -1,4 +1,4 @@ -import { getDeadlineNearThresholdMin } from "../../../config.js"; +import { DEADLINE_NEAR_THRESHOLD_MIN } from "../constants.js"; import { analyzeNicknameIssues } from "../onboarding-templates.js"; import type { OnboardingSession } from "../types.js"; import type { OnboardingPhase, RemainingStep } from "./onboarding-state.js"; @@ -30,7 +30,7 @@ function isDeadlineNear(session: OnboardingSession | null): boolean { return false; } - return deadlineAt - Date.now() <= getDeadlineNearThresholdMin() * 60_000; + return deadlineAt - Date.now() <= DEADLINE_NEAR_THRESHOLD_MIN * 60_000; } export function derivePhaseFromSession( diff --git a/src/tools/builtins/check-onboarding-progress.ts b/src/tools/builtins/check-onboarding-progress.ts new file mode 100644 index 0000000..0772a85 --- /dev/null +++ b/src/tools/builtins/check-onboarding-progress.ts @@ -0,0 +1,64 @@ +import { z } from "zod"; +import { getOnboardingSessionKeyPrefix } from "../../config.js"; +import type { OnboardingSession } from "@skills/onboarding/types"; +import type { Tool } from "../tool.js"; + +const argsSchema = z.object({ + chatroomId: z.string().min(1), + userId: z.string().min(1), +}); + +const resultSchema = z.object({ + nicknameDone: z.boolean(), + photoDone: z.boolean(), + remainingSteps: z.array(z.enum(["nickname", "photo"])), + deadlineInHours: z.number().nullable(), +}); + +function getRemainingSteps(session: OnboardingSession | null): Array<"nickname" | "photo"> { + if (!session) { + return ["nickname", "photo"]; + } + + const remaining: Array<"nickname" | "photo"> = []; + if (!session.nicknameVerified) { + remaining.push("nickname"); + } + if (!session.photoVerified) { + remaining.push("photo"); + } + return remaining; +} + +export const checkOnboardingProgressTool: Tool< + z.infer, + z.infer +> = { + name: "check-onboarding-progress", + description: + "현재 유저의 온보딩 진행 상태를 읽어 남은 절차와 데드라인까지 남은 시간을 구조화해서 반환한다.", + argsSchema, + resultSchema, + async execute(args, ctx) { + const session = + ctx.session && ctx.chatroomId === args.chatroomId && ctx.userId === args.userId + ? ctx.session + : ctx.garnetClient + ? await ctx.garnetClient.getJson( + `${getOnboardingSessionKeyPrefix()}:onboarding:${args.chatroomId}:${args.userId}`, + ) + : null; + + const deadlineAt = session?.deadlineAt ? Date.parse(session.deadlineAt) : NaN; + const deadlineInHours = Number.isFinite(deadlineAt) + ? Math.max(0, (deadlineAt - Date.now()) / 3_600_000) + : null; + + return { + nicknameDone: session?.nicknameVerified ?? false, + photoDone: session?.photoVerified ?? false, + remainingSteps: getRemainingSteps(session), + deadlineInHours, + }; + }, +}; diff --git a/src/tools/builtins/get-room-context.ts b/src/tools/builtins/get-room-context.ts new file mode 100644 index 0000000..278ca02 --- /dev/null +++ b/src/tools/builtins/get-room-context.ts @@ -0,0 +1,48 @@ +import { z } from "zod"; +import type { Tool } from "../tool.js"; + +const argsSchema = z.object({ + chatroomId: z.string().min(1), + limit: z.number().int().min(1).max(20).optional(), +}); + +const roomMessageSchema = z.object({ + senderId: z.string(), + nickname: z.string(), + message: z.string(), + timestamp: z.string(), +}); + +const resultSchema = z.object({ + messages: z.array(roomMessageSchema), +}); + +export const getRoomContextTool: Tool< + z.infer, + z.infer +> = { + name: "get-room-context", + description: + "최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.", + argsSchema, + resultSchema, + async execute(args, ctx) { + if (!ctx.roomContextService) { + return { messages: [] }; + } + + const messages = await ctx.roomContextService.getRecentMessages( + args.chatroomId, + args.limit, + ); + + return { + messages: messages.map((message) => ({ + senderId: message.senderId, + nickname: message.nickname, + message: message.message, + timestamp: message.timestamp.toISOString(), + })), + }; + }, +}; diff --git a/src/tools/builtins/index.ts b/src/tools/builtins/index.ts new file mode 100644 index 0000000..7e40a78 --- /dev/null +++ b/src/tools/builtins/index.ts @@ -0,0 +1,20 @@ +import { ToolRegistry } from "../registry.js"; +import { checkOnboardingProgressTool } from "./check-onboarding-progress.js"; +import { getRoomContextTool } from "./get-room-context.js"; +import { lookupCommunityRuleTool } from "./lookup-community-rule.js"; +import { validateNicknameTool } from "./validate-nickname.js"; + +export function createBuiltInToolRegistry(): ToolRegistry { + return new ToolRegistry() + .register(validateNicknameTool) + .register(checkOnboardingProgressTool) + .register(lookupCommunityRuleTool) + .register(getRoomContextTool); +} + +export { + checkOnboardingProgressTool, + getRoomContextTool, + lookupCommunityRuleTool, + validateNicknameTool, +}; diff --git a/src/tools/builtins/lookup-community-rule.ts b/src/tools/builtins/lookup-community-rule.ts new file mode 100644 index 0000000..990d14d --- /dev/null +++ b/src/tools/builtins/lookup-community-rule.ts @@ -0,0 +1,54 @@ +import { z } from "zod"; +import { + DEADLINE_NEAR_THRESHOLD_MIN, + DISCORD_GUIDE_TEXT, + HEART_RULE_TEXT, + MIN_AGE, + NICKNAME_FORMAT_TEXT, + TIER_CODES_TEXT, +} from "@skills/onboarding/constants"; +import type { Tool } from "../tool.js"; + +const argsSchema = z.object({ + topic: z.enum([ + "age", + "tier-codes", + "discord-link", + "heart-rule", + "nickname-format", + "deadline", + ]), +}); + +const resultSchema = z.object({ + topic: argsSchema.shape.topic, + text: z.string(), + source: z.literal("constants"), +}); + +const TOPIC_TEXT: Record["topic"], string> = { + age: `가입 가능 연령은 ${MIN_AGE}세 이상이에요.`, + "tier-codes": `사용 가능한 티어 코드는 ${TIER_CODES_TEXT}예요.`, + "discord-link": DISCORD_GUIDE_TEXT, + "heart-rule": HEART_RULE_TEXT, + "nickname-format": NICKNAME_FORMAT_TEXT, + deadline: `데드라인이 ${DEADLINE_NEAR_THRESHOLD_MIN}분 이하로 남으면 마감이 임박한 상태로 안내해요.`, +}; + +export const lookupCommunityRuleTool: Tool< + z.infer, + z.infer +> = { + name: "lookup-community-rule", + description: + "나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 상수에서 조회한다.", + argsSchema, + resultSchema, + execute(args) { + return Promise.resolve({ + topic: args.topic, + text: TOPIC_TEXT[args.topic], + source: "constants", + }); + }, +}; diff --git a/src/tools/builtins/validate-nickname.ts b/src/tools/builtins/validate-nickname.ts new file mode 100644 index 0000000..f7bec97 --- /dev/null +++ b/src/tools/builtins/validate-nickname.ts @@ -0,0 +1,52 @@ +import { z } from "zod"; +import { analyzeNicknameIssues } from "@skills/onboarding/onboarding-templates"; +import type { Tool } from "../tool.js"; + +const argsSchema = z.object({ + nickname: z.string().min(1), +}); + +const nicknameIssueSchema = z.object({ + kind: z.enum([ + "missing-tier", + "invalid-age", + "missing-gender", + "wrong-separator", + "unknown-tier", + "other", + ]), + message: z.string(), + suggestion: z.string().optional(), +}); + +const resultSchema = z.object({ + valid: z.boolean(), + issues: z.array(nicknameIssueSchema), + normalized: z.string().optional(), +}); + +export const validateNicknameTool: Tool< + z.infer, + z.infer +> = { + name: "validate-nickname", + description: + "새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.", + argsSchema, + resultSchema, + execute(args) { + const normalized = args.nickname.trim(); + const issues = analyzeNicknameIssues(normalized); + + const result: z.infer = { + valid: issues.length === 0, + issues, + }; + + if (issues.length === 0) { + result.normalized = normalized; + } + + return Promise.resolve(result); + }, +}; diff --git a/src/tools/index.ts b/src/tools/index.ts new file mode 100644 index 0000000..1d8d439 --- /dev/null +++ b/src/tools/index.ts @@ -0,0 +1,8 @@ +export { ToolRegistry } from "./registry.js"; +export { parseToolCalls } from "./parser.js"; +export { + serializeToolsForHermes, + serializeToolsForOpenAi, +} from "./serializer.js"; +export type { OpenAiToolDefinition } from "./serializer.js"; +export type { Tool, ToolContext } from "./tool.js"; diff --git a/src/tools/parser.test.ts b/src/tools/parser.test.ts new file mode 100644 index 0000000..820a1ca --- /dev/null +++ b/src/tools/parser.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { parseToolCalls } from "./parser.js"; + +describe("parseToolCalls", () => { + it("parses Hermes tool call blocks", () => { + const result = parseToolCalls( + '{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여/D"}}', + ); + + expect(result).toEqual([ + { + name: "validate-nickname", + arguments: { + nickname: "프리리/95/여/D", + }, + }, + ]); + }); + + it("parses OpenAI tool_calls arrays", () => { + const result = parseToolCalls({ + tool_calls: [ + { + id: "call_1", + function: { + name: "lookup-community-rule", + arguments: '{"topic":"age"}', + }, + }, + ], + }); + + expect(result).toEqual([ + { + id: "call_1", + name: "lookup-community-rule", + arguments: { + topic: "age", + }, + }, + ]); + }); + + it("parses tool calls nested in choices[0].message", () => { + const result = parseToolCalls({ + choices: [ + { + message: { + tool_calls: [ + { + id: "call_2", + function: { + name: "check-onboarding-progress", + arguments: '{"chatroomId":"room-1","userId":"user-1"}', + }, + }, + ], + }, + }, + ], + }); + + expect(result[0]?.name).toBe("check-onboarding-progress"); + expect(result[0]?.arguments).toEqual({ + chatroomId: "room-1", + userId: "user-1", + }); + }); +}); diff --git a/src/tools/parser.ts b/src/tools/parser.ts new file mode 100644 index 0000000..113a5da --- /dev/null +++ b/src/tools/parser.ts @@ -0,0 +1,136 @@ +export interface ParsedToolCall { + id?: string | undefined; + name: string; + arguments: Record; +} + +interface OpenAiToolCallShape { + id?: string; + function?: { + name?: string; + arguments?: string; + }; +} + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null + ? (value as Record) + : {}; +} + +function parseJsonArguments(raw: unknown): Record | null { + if (typeof raw === "string") { + try { + const parsed = JSON.parse(raw) as unknown; + return asRecord(parsed); + } catch { + return null; + } + } + + if (typeof raw === "object" && raw !== null) { + return raw as Record; + } + + return null; +} + +function parseHermesCalls(content: string): ParsedToolCall[] { + const matches = [...content.matchAll(/\s*([\s\S]*?)\s*<\/tool_call>/gu)]; + + return matches.flatMap((match) => { + const payload = match[1]; + if (!payload) { + return []; + } + + try { + const parsed = JSON.parse(payload) as + | { + id?: string; + name?: string; + arguments?: unknown; + } + | Array<{ + id?: string; + name?: string; + arguments?: unknown; + }>; + + const items = Array.isArray(parsed) ? parsed : [parsed]; + return items.flatMap((item) => { + const args = parseJsonArguments(item.arguments); + if (!item.name || !args) { + return []; + } + + return [ + { + id: item.id, + name: item.name, + arguments: args, + }, + ]; + }); + } catch { + return []; + } + }); +} + +function parseOpenAiCalls(value: unknown): ParsedToolCall[] { + const record = asRecord(value); + const choices = Array.isArray(record["choices"]) + ? (record["choices"] as unknown[]) + : []; + const firstChoice = choices[0]; + const candidates = [ + record["tool_calls"], + asRecord(record["message"])["tool_calls"], + asRecord(asRecord(firstChoice)["message"])["tool_calls"], + ]; + + const toolCallsCandidate = candidates.find(Array.isArray); + if (!Array.isArray(toolCallsCandidate)) { + return []; + } + + return toolCallsCandidate.flatMap((item) => { + const call = item as OpenAiToolCallShape; + const args = parseJsonArguments(call.function?.arguments); + const name = call.function?.name; + + if (!name || !args) { + return []; + } + + return [ + { + id: call.id, + name, + arguments: args, + }, + ]; + }); +} + +export function parseToolCalls(value: unknown): ParsedToolCall[] { + if (typeof value === "string") { + return parseHermesCalls(value); + } + + const record = asRecord(value); + const content = + typeof record["content"] === "string" + ? record["content"] + : typeof asRecord(record["message"])["content"] === "string" + ? (asRecord(record["message"])["content"] as string) + : ""; + + const parsedOpenAiCalls = parseOpenAiCalls(value); + if (parsedOpenAiCalls.length > 0) { + return parsedOpenAiCalls; + } + + return content ? parseHermesCalls(content) : []; +} diff --git a/src/tools/registry.ts b/src/tools/registry.ts new file mode 100644 index 0000000..5539026 --- /dev/null +++ b/src/tools/registry.ts @@ -0,0 +1,32 @@ +import type { Tool } from "./tool.js"; + +export class ToolRegistry { + private readonly tools = new Map>(); + + register(tool: Tool): this { + this.tools.set(tool.name, tool as Tool); + return this; + } + + get(name: string): Tool { + const tool = this.tools.get(name); + if (!tool) { + throw new Error(`Unknown tool: ${name}`); + } + return tool; + } + + has(name: string): boolean { + return this.tools.has(name); + } + + list(): Tool[] { + return [...this.tools.values()]; + } + + pick(names: readonly string[]): Tool[] { + return names + .filter((name) => this.tools.has(name)) + .map((name) => this.get(name)); + } +} diff --git a/src/tools/serializer.ts b/src/tools/serializer.ts new file mode 100644 index 0000000..25c967a --- /dev/null +++ b/src/tools/serializer.ts @@ -0,0 +1,42 @@ +import { z } from "zod"; +import type { Tool } from "./tool.js"; + +export interface OpenAiToolDefinition { + type: "function"; + function: { + name: string; + description: string; + parameters: unknown; + }; +} + +function toToolDescriptor(tool: Tool): { + name: string; + description: string; + parameters: unknown; +} { + return { + name: tool.name, + description: tool.description, + parameters: z.toJSONSchema(tool.argsSchema), + }; +} + +export function serializeToolsForHermes( + tools: readonly Tool[], +): string { + const descriptors = tools.map(toToolDescriptor); + return `\n${JSON.stringify(descriptors, null, 2)}\n`; +} + +export function serializeToolsForOpenAi( + tools: readonly Tool[], +): OpenAiToolDefinition[] { + return tools.map((tool) => { + const descriptor = toToolDescriptor(tool); + return { + type: "function", + function: descriptor, + }; + }); +} diff --git a/src/tools/tool.ts b/src/tools/tool.ts new file mode 100644 index 0000000..bfcbdf6 --- /dev/null +++ b/src/tools/tool.ts @@ -0,0 +1,20 @@ +import type { GarnetClient } from "@common/garnet"; +import type { RoomContextService } from "@room/room-context-service"; +import type { ZodType } from "zod"; +import type { OnboardingSession } from "@skills/onboarding/types"; + +export interface ToolContext { + chatroomId?: string | undefined; + userId?: string | undefined; + session?: OnboardingSession | null | undefined; + garnetClient?: GarnetClient | undefined; + roomContextService?: RoomContextService | undefined; +} + +export interface Tool { + name: string; + description: string; + argsSchema: ZodType; + resultSchema: ZodType; + execute(args: A, ctx: ToolContext): Promise; +} diff --git a/src/workers/background.ts b/src/workers/background.ts new file mode 100644 index 0000000..9549c38 --- /dev/null +++ b/src/workers/background.ts @@ -0,0 +1,207 @@ +import { loadEnvironmentFromNodeEnv } from "../common/env-loader.js"; +import { + getOpenRouterApiUrl, + getOpenRouterEnabled, +} from "../config.js"; + +loadEnvironmentFromNodeEnv(); + +const baseServiceName = process.env["SERVICE_NAME"]?.trim() || "lolympus-agent"; +process.env["SERVICE_NAME"] = `${baseServiceName}-worker`; + +const { getLogger, initLogger } = await import("../observability/logger.js"); +initLogger(); +const logger = getLogger(); +const { initTracing, shutdownTracing } = await import( + "../observability/tracing.js" +); +initTracing(); + +const { GarnetClient } = await import("../common/garnet.js"); +const { NatsClient } = await import("../common/nats.js"); +const { LlmClient } = await import("../llm/index.js"); +const { getRawSecret } = await import("../common/secret-manager.js"); +const { ConversationStore } = await import("../conversation/index.js"); +const { ShadowWorker } = await import("../eval/shadow-worker.js"); +const { AlertService } = await import("../observability/alert.js"); + +function firstNonEmpty( + ...values: Array +): string | undefined { + for (const value of values) { + if (value?.trim()) { + return value.trim(); + } + } + + return undefined; +} + +function parseEndpointList( + ...values: Array +): string[] { + const items: string[] = []; + + for (const value of values) { + if (!value?.trim()) { + continue; + } + + for (const item of value.split(/[\n,]/u)) { + const trimmed = item.trim(); + if (trimmed) { + items.push(trimmed); + } + } + } + + return [...new Set(items)]; +} + +function toProviderEndpoints( + urls: string[], + provider?: "llama" | "cliproxy" | "openrouter", +): Array { + if (!provider) { + return urls; + } + + return urls.map((url) => ({ url, provider })); +} + +function installShutdownHooks( + shutdown: (reason: string, error?: unknown) => Promise, +): void { + let shuttingDown = false; + + const runShutdown = async ( + reason: string, + error?: unknown, + ): Promise => { + if (shuttingDown) { + return; + } + shuttingDown = true; + + if (error) { + logger.error({ err: error }, `[Background] ${reason}`); + } else { + logger.info(`[Background] ${reason}`); + } + + try { + await shutdown(reason, error); + } catch (shutdownError) { + logger.error({ err: shutdownError }, "[Background] Cleanup failed"); + } finally { + process.exit(error ? 1 : 0); + } + }; + + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + void runShutdown(signal); + }); + } + + process.once("uncaughtException", (error) => { + void runShutdown("uncaughtException", error); + }); + + process.once("unhandledRejection", (reason) => { + void runShutdown("unhandledRejection", reason); + }); +} + +async function bootstrap(): Promise { + const serviceName = process.env["SERVICE_NAME"] ?? "lolympus-agent-worker"; + + const garnetClient = new GarnetClient(); + await garnetClient.connect(); + const natsClient = new NatsClient(); + await natsClient.connect(); + + const cliproxyEndpoints = parseEndpointList( + getRawSecret("CLI_PROXY_API_URLS"), + process.env["CLI_PROXY_API_URLS"], + ); + const generalLlmEndpoints = parseEndpointList( + getRawSecret("AGENT_LLM_API_URLS"), + process.env["AGENT_LLM_API_URLS"], + getRawSecret("LLAMA_SERVER_API_URL"), + process.env["LLAMA_SERVER_API_URL"], + ); + const openRouterEndpoints = getOpenRouterEnabled() + ? parseEndpointList( + getRawSecret("OPENROUTER_API_URL"), + process.env["OPENROUTER_API_URL"], + getOpenRouterApiUrl(), + ) + : []; + const llmApiKey = firstNonEmpty( + getRawSecret("AGENT_LLM_API_KEY"), + process.env["AGENT_LLM_API_KEY"], + getRawSecret("LLAMA_SERVER_API_KEY"), + process.env["LLAMA_SERVER_API_KEY"], + ); + const providerApiKeys = Object.fromEntries( + [ + [ + "cliproxy", + firstNonEmpty( + getRawSecret("CLI_PROXY_API_KEY"), + process.env["CLI_PROXY_API_KEY"], + ), + ], + ["llama", llmApiKey], + [ + "openrouter", + firstNonEmpty( + getRawSecret("OPENROUTER_API_KEY"), + process.env["OPENROUTER_API_KEY"], + ), + ], + ].filter((entry): entry is [string, string] => typeof entry[1] === "string"), + ) as Partial>; + + const llmClient = new LlmClient({ + endpoints: [ + ...toProviderEndpoints(cliproxyEndpoints, "cliproxy"), + ...toProviderEndpoints(openRouterEndpoints, "openrouter"), + ...toProviderEndpoints(generalLlmEndpoints, "llama"), + ], + garnetClient, + apiKey: llmApiKey, + model: firstNonEmpty( + getRawSecret("AGENT_LLM_MODEL"), + process.env["AGENT_LLM_MODEL"], + ), + providerApiKeys, + }); + const conversationStore = new ConversationStore(garnetClient); + const alertService = new AlertService(garnetClient, natsClient); + const worker = new ShadowWorker( + garnetClient, + llmClient, + conversationStore, + alertService, + ); + + installShutdownHooks(async () => { + await worker.stop(); + await natsClient.close(); + await garnetClient.close(); + await shutdownTracing(); + }); + + await worker.start(); + logger.info({ serviceName }, "✓ Background worker bootstrapped"); +} + +try { + await bootstrap(); +} catch (error) { + logger.error({ err: error }, "[Background] Fatal startup error"); + await shutdownTracing(); + process.exitCode = 1; +} diff --git a/tsconfig.alias.json b/tsconfig.alias.json new file mode 100644 index 0000000..07bf2ee --- /dev/null +++ b/tsconfig.alias.json @@ -0,0 +1,19 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "paths": { + "@agent": ["src/agent/index"], + "@agent/*": ["src/agent/*"], + "@skills/*": ["src/skills/*"], + "@llm": ["src/llm/index"], + "@llm/*": ["src/llm/*"], + "@obs/*": ["src/observability/*"], + "@eval/*": ["src/eval/*"], + "@tools/*": ["src/tools/*"], + "@common/*": ["src/common/*"], + "@conv/*": ["src/conversation/*"], + "@room": ["src/room-context/index"], + "@room/*": ["src/room-context/*"] + } + } +}