Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+18
View File
@@ -0,0 +1,18 @@
module.exports = {
apps: [
{
name: "agent",
script: "dist/main.js",
instances: 1,
max_memory_restart: "512M",
kill_timeout: 12000,
},
{
name: "agent-worker",
script: "dist/workers/background.js",
instances: 1,
max_memory_restart: "512M",
kill_timeout: 12000,
},
],
};
+1 -1
View File
@@ -65,7 +65,7 @@ Get-ChildItem Env: | Where-Object { $_.Name -like "*AWS*" -or $_.Name -eq "NODE_
if ($UsePm2) {
pm2 start dist/main.js --name $Pm2Name --update-env
pm2 start ecosystem.config.cjs --only agent,agent-worker --update-env
} else {
node ./dist/main.js
}
+2 -1
View File
@@ -4,7 +4,7 @@
"main": "dist/main.js",
"type": "module",
"scripts": {
"build": "tsc && tsc-alias",
"build": "tsc && tsc-alias -p tsconfig.alias.json --resolve-full-paths --resolve-full-extension .js",
"typecheck": "tsc --noEmit",
"test": "vitest run",
"test:watch": "vitest",
@@ -14,6 +14,7 @@
"format": "prettier --write .",
"eval": "node --import tsx src/eval/cli.ts run --set=golden",
"eval:capture": "node --import tsx src/eval/cli.ts capture",
"eval:baseline": "node --import tsx src/eval/cli.ts baseline",
"start:dev": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'dev'); \"./launch-server.ps1\"\"",
"start:test": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'test'); \"./launch-server.ps1\"\"",
"start:prod": "pwsh -Command \"[System.Environment]::SetEnvironmentVariable('NODE_ENV', 'prod'); \"./launch-server.ps1\"\"",
+8 -1
View File
@@ -6,7 +6,11 @@ import { getLogger } from "@obs/logger";
import { withSkillSpan, withSpan } from "@obs/spans";
import type { Skill } from "@skills/skill";
import type { AgentRequest, AgentResponse } from "./types.js";
import { getRequestSubject } from "../config.js";
import {
getAgentRequestTimeoutMs,
getRequestSubject,
getSafetyMarginMs,
} from "../config.js";
function isValidRequest(data: unknown): data is AgentRequest {
if (typeof data !== "object" || data === null) return false;
@@ -64,6 +68,9 @@ export class Agent {
await runWithContext(
{
chatroomId: request.chatroomId,
deadlineAt: new Date(
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
).toISOString(),
requestId: request.requestId,
skill: request.skill,
startedAt: new Date().toISOString(),
+232
View File
@@ -4,6 +4,37 @@ import { getLogger } from "@obs/logger";
type GarnetRedisClient = ReturnType<typeof createClient>;
export interface StreamEntry {
deliveriesCounter?: number | undefined;
id: string;
message: Record<string, string>;
millisElapsedFromDelivery?: number | undefined;
}
export interface StreamReadResult {
messages: StreamEntry[];
name: string;
}
export interface StreamPendingEntry {
consumer: string;
deliveriesCounter: number;
id: string;
millisecondsSinceLastDelivery: number;
}
interface RawStreamMessage {
deliveriesCounter?: number | undefined;
id: string;
message: unknown;
millisElapsedFromDelivery?: number | undefined;
}
interface RawStreamReadResult {
messages: RawStreamMessage[];
name: string;
}
function normalizeGarnetUrl(endpoint: string): string {
const trimmed = endpoint.trim();
if (!trimmed) {
@@ -17,6 +48,29 @@ function normalizeGarnetUrl(endpoint: string): string {
return `redis://${trimmed}`;
}
function normalizeStreamFields(
fields: unknown,
): Record<string, string> {
if (fields instanceof Map) {
return Object.fromEntries(
[...fields.entries()].map(([key, value]) => [
String(key),
String(value),
]),
);
}
if (typeof fields === "object" && fields !== null) {
return Object.fromEntries(
Object.entries(fields as Record<string, unknown>).map(
([key, value]) => [key, String(value)],
),
);
}
return {};
}
export class GarnetClient {
private client: GarnetRedisClient | null = null;
private readonly logger = getLogger();
@@ -98,6 +152,27 @@ export class GarnetClient {
await client.set(key, value);
}
async setStringIfAbsent(
key: string,
value: string,
ttlSeconds?: number,
): Promise<boolean> {
const client = this.getClient();
const setOptions: {
EX?: number;
NX: true;
} = {
NX: true,
};
if (ttlSeconds && ttlSeconds > 0) {
setOptions.EX = ttlSeconds;
}
const result = await client.set(key, value, {
...setOptions,
});
return result === "OK";
}
async delete(key: string): Promise<number> {
return this.getClient().del(key);
}
@@ -132,6 +207,163 @@ export class GarnetClient {
return this.getClient().xAdd(key, "*", fields, trimOptions);
}
async createConsumerGroup(
key: string,
group: string,
startId = "$",
): Promise<void> {
await this.getClient().xGroupCreate(key, group, startId, {
MKSTREAM: true,
});
}
async readGroup(
key: string,
group: string,
consumer: string,
startId: string,
options?: {
blockMs?: number | undefined;
count?: number | undefined;
},
): Promise<StreamReadResult[] | null> {
const readOptions: {
BLOCK?: number;
COUNT?: number;
} = {};
if (options?.blockMs !== undefined) {
readOptions.BLOCK = options.blockMs;
}
if (options?.count !== undefined) {
readOptions.COUNT = options.count;
}
const reply = await this.getClient().xReadGroup(
group,
consumer,
{
key,
id: startId,
},
readOptions,
);
if (!reply) {
return null;
}
return (reply as RawStreamReadResult[]).map((stream) => ({
name: String(stream.name),
messages: stream.messages.map((message) => ({
deliveriesCounter:
typeof message.deliveriesCounter === "number"
? message.deliveriesCounter
: undefined,
id: String(message.id),
message: normalizeStreamFields(message.message),
millisElapsedFromDelivery:
typeof message.millisElapsedFromDelivery === "number"
? message.millisElapsedFromDelivery
: undefined,
})),
}));
}
async acknowledgeStream(
key: string,
group: string,
ids: string | string[],
): Promise<number> {
return this.getClient().xAck(key, group, ids);
}
async getPendingStreamRange(
key: string,
group: string,
count: number,
options?: {
consumer?: string | undefined;
idleMs?: number | undefined;
start?: string | undefined;
end?: string | undefined;
},
): Promise<StreamPendingEntry[]> {
const pendingOptions: {
IDLE?: number;
consumer?: string;
} = {};
if (options?.idleMs !== undefined) {
pendingOptions.IDLE = options.idleMs;
}
if (options?.consumer !== undefined) {
pendingOptions.consumer = options.consumer;
}
const reply = await this.getClient().xPendingRange(
key,
group,
options?.start ?? "-",
options?.end ?? "+",
count,
pendingOptions,
);
return reply.map((entry) => ({
consumer: String(entry.consumer),
deliveriesCounter: entry.deliveriesCounter,
id: String(entry.id),
millisecondsSinceLastDelivery: entry.millisecondsSinceLastDelivery,
}));
}
async claimPendingStream(
key: string,
group: string,
consumer: string,
minIdleTimeMs: number,
ids: string | string[],
options?: {
retryCount?: number | undefined;
},
): Promise<StreamEntry[]> {
const claimOptions: {
RETRYCOUNT?: number;
} = {};
if (options?.retryCount !== undefined) {
claimOptions.RETRYCOUNT = options.retryCount;
}
const claimed = await this.getClient().xClaim(
key,
group,
consumer,
minIdleTimeMs,
ids,
claimOptions,
);
return claimed.flatMap((message) => {
if (!message) {
return [];
}
return [
{
deliveriesCounter:
typeof message.deliveriesCounter === "number"
? message.deliveriesCounter
: undefined,
id: String(message.id),
message: normalizeStreamFields(message.message),
millisElapsedFromDelivery:
typeof message.millisElapsedFromDelivery === "number"
? message.millisElapsedFromDelivery
: undefined,
},
];
});
}
async scanKeys(pattern: string): Promise<string[]> {
const keys: string[] = [];
+19
View File
@@ -0,0 +1,19 @@
import type { EvalReport } from "@eval/types";
export const AGENT_REQUEST_SUBJECT = "lolympus-agent.request";
export const AGENT_ALERT_SUBJECT = "lolympus-agent.alert";
export const AGENT_EVAL_REPORT_SUBJECT = "lolympus-agent.eval.report";
export interface AlertPayload {
kind: "latency" | "eval-score-drop" | "llm-error-rate" | "endpoint-down";
metric: string;
severity: "info" | "warn" | "critical";
suggestedAction?: string | undefined;
threshold: number;
timestamp: string;
traceId?: string | undefined;
value: number;
window: string;
}
export type EvalReportPayload = EvalReport;
+146 -2
View File
@@ -65,8 +65,12 @@ export function getOtelSamplingRatio(): number {
return getRatioNumber("OTEL_SAMPLING_RATIO", fallback);
}
export function getOtelExporter(): "stdout" {
return "stdout";
export function getOtelExporter(): "stdout" | "none" {
const env = process.env["OTEL_EXPORTER"]?.trim().toLowerCase();
if (env === "none" || env === "stdout") {
return env;
}
return getNodeEnv() === "dev" ? "none" : "stdout";
}
export function getRequestSubject(): string {
@@ -75,6 +79,10 @@ export function getRequestSubject(): string {
);
}
export function getAgentRequestTimeoutMs(): number {
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
}
export function getOnboardingSessionKeyPrefix(): string {
return process.env["ONBOARDING_SESSION_KEY_PREFIX"]?.trim() || "lolympus";
}
@@ -95,6 +103,10 @@ export function getBackgroundStreamMaxLen(): number {
return getPositiveNumber("BACKGROUND_STREAM_MAXLEN", 5_000);
}
export function getEvalStreamMaxLen(): number {
return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000);
}
export function getMaxConversationMessages(): number {
return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20);
}
@@ -123,14 +135,110 @@ export function getClassifierMaxTokens(): number {
return getPositiveNumber("CLASSIFIER_MAX_TOKENS", 16);
}
export function getLlmJudgeModel(): string {
return process.env["LLM_JUDGE_MODEL"]?.trim() || "gpt-5.3-codex-spark";
}
export function getLlmJudgeMaxTokens(): number {
return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128);
}
export function getSafetyMarginMs(): number {
return getPositiveNumber("SAFETY_MARGIN_MS", 500);
}
export function getDeadlineNearThresholdMin(): number {
return getPositiveNumber("DEADLINE_NEAR_THRESHOLD_MIN", 30);
}
export function getOpenRouterEnabled(): boolean {
const value = process.env["OPENROUTER_ENABLED"]?.trim().toLowerCase();
return value === "1" || value === "true" || value === "yes";
}
export function getOpenRouterApiUrl(): string {
return process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1";
}
export function getTier0Model(): string {
return (
process.env["TIER0_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.3-codex-spark"
);
}
export function getTier1PlannerModel(): string {
return (
process.env["TIER1_PLANNER_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.4"
);
}
export function getTier1ResponderModel(): string {
return (
process.env["TIER1_RESPONDER_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.3-codex-spark"
);
}
export function getTier2CriticModel(): string {
return (
process.env["TIER2_CRITIC_MODEL"]?.trim() ||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
"gpt-5.4"
);
}
export function getOpenRouterTier1Model(): string {
return (
process.env["OPENROUTER_TIER1_MODEL"]?.trim() ||
"openai/gpt-5.3-codex-spark"
);
}
export function getOpenRouterTier2Model(): string {
return process.env["OPENROUTER_TIER2_MODEL"]?.trim() || "openai/gpt-5.4";
}
export function getLlmMaxTokens(): number {
return getPositiveNumber("LLM_MAX_TOKENS", 768);
}
export function getTier0MaxTokens(): number {
return getPositiveNumber("TIER0_MAX_TOKENS", 384);
}
export function getTier1MaxTokens(): number {
return getPositiveNumber("TIER1_MAX_TOKENS", 512);
}
export function getTier2MaxTokens(): number {
return getPositiveNumber("TIER2_MAX_TOKENS", 512);
}
export function getPlannerMaxTokens(): number {
return getPositiveNumber("PLANNER_MAX_TOKENS", 256);
}
export function getCriticMaxTokens(): number {
return getPositiveNumber("CRITIC_MAX_TOKENS", 512);
}
export function getTier0EstimatedCallMs(): number {
return getPositiveNumber("TIER0_ESTIMATED_CALL_MS", 600);
}
export function getTier1EstimatedCallMs(): number {
return getPositiveNumber("TIER1_ESTIMATED_CALL_MS", 1200);
}
export function getTier2EstimatedCallMs(): number {
return getPositiveNumber("TIER2_ESTIMATED_CALL_MS", 1400);
}
export function getLlmTemperature(): number {
const raw = process.env["LLM_TEMPERATURE"]?.trim();
if (raw) {
@@ -151,3 +259,39 @@ export function getMongoDbProjectId(): string | undefined {
export function getRoomContextLimit(): number {
return getPositiveNumber("ROOM_CONTEXT_LIMIT", 8);
}
export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
switch (tier) {
case 0:
return getRatioNumber(
"SHADOW_EVAL_SAMPLING_RATE_TIER0",
getNodeEnv() === "prod" ? 0.1 : 1.0,
);
case 1:
return getRatioNumber(
"SHADOW_EVAL_SAMPLING_RATE_TIER1",
getNodeEnv() === "prod" ? 0.5 : 1.0,
);
case 2:
return getRatioNumber(
"SHADOW_EVAL_SAMPLING_RATE_TIER2",
1.0,
);
}
}
export function getAlertThresholdP95LatencyMs(): number {
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
}
export function getAlertThresholdEvalScoreDropPct(): number {
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
}
export function getAlertThresholdLlmErrorRate(): number {
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
}
export function getAlertRateLimitSec(): number {
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
}
+19 -11
View File
@@ -1,5 +1,6 @@
import type { ConversationMessage } from "@agent/types";
import type { GarnetClient } from "@common/garnet";
import { getRequestLogger } from "@obs/context";
import {
getBackgroundStreamMaxLen,
getConversationTtlSeconds,
@@ -52,17 +53,24 @@ export class ConversationStore {
await this.garnetClient.setJson(key, trimmed, getConversationTtlSeconds());
if (trimmed.length > 5) {
await this.garnetClient.appendStream(
BACKGROUND_STREAM_KEY,
{
type: "summarize",
chatroomId,
userId,
},
{
maxLen: getBackgroundStreamMaxLen(),
},
);
void this.garnetClient
.appendStream(
BACKGROUND_STREAM_KEY,
{
type: "summarize",
chatroomId,
userId,
},
{
maxLen: getBackgroundStreamMaxLen(),
},
)
.catch((error) => {
getRequestLogger().warn(
{ err: error, chatroomId, userId },
"[ConversationStore] Failed to publish summarize job",
);
});
}
}
+1 -1
View File
@@ -8,7 +8,7 @@ function formatHistory(history: ConversationMessage[]): string {
export async function summarizeConversation(
llmClient: LlmClient,
history: ConversationMessage[],
existingSummary?: string | undefined,
existingSummary?: string,
): Promise<string> {
if (!llmClient.hasEndpoints() || history.length === 0) {
return existingSummary ?? "";
+166 -17
View File
@@ -1,9 +1,3 @@
function printPhaseNotice(command: string): void {
console.info(
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
);
}
function parsePositiveInteger(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
@@ -13,26 +7,181 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
interface ParsedArgs {
command: string;
flags: Map<string, string | true>;
positionals: string[];
}
function parseArgs(argv: string[]): ParsedArgs {
const [command = "run", ...rest] = argv;
const flags = new Map<string, string | true>();
const positionals: string[] = [];
for (let index = 0; index < rest.length; index++) {
const token = rest[index];
if (!token) {
continue;
}
if (!token.startsWith("--")) {
positionals.push(token);
continue;
}
const withoutPrefix = token.slice(2);
const equalsIndex = withoutPrefix.indexOf("=");
if (equalsIndex >= 0) {
const key = withoutPrefix.slice(0, equalsIndex);
const value = withoutPrefix.slice(equalsIndex + 1);
flags.set(key, value || true);
continue;
}
const next = rest[index + 1];
if (next && !next.startsWith("--")) {
flags.set(withoutPrefix, next);
index += 1;
continue;
}
flags.set(withoutPrefix, true);
}
return {
command,
flags,
positionals,
};
}
function getFlagString(
flags: Map<string, string | true>,
key: string,
): string | undefined {
const value = flags.get(key);
return typeof value === "string" ? value : undefined;
}
function hasFlag(
flags: Map<string, string | true>,
key: string,
): boolean {
return flags.has(key);
}
function printRunSummary(report: {
generatedAt: string;
summary: {
cases: number;
passRate: number;
ruleAverageScore: number;
averageLatencyMs: number;
llmAverageOverall: number | null;
llmJudgedCases: number;
};
baselineDelta?: {
passRate: number | null;
ruleAverageScore: number | null;
llmAverageOverall: number | null;
averageLatencyMs: number | null;
} | undefined;
}, reportPath: string): void {
const llmLine =
report.summary.llmAverageOverall === null
? "n/a"
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
console.info(`[eval] generatedAt=${report.generatedAt}`);
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
console.info(
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
);
console.info(`[eval] llmAverageOverall=${llmLine}`);
if (report.baselineDelta) {
console.info(
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
);
}
console.info(`[eval] report=${reportPath}`);
}
async function main(): Promise<void> {
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
loadEnvironmentFromNodeEnv();
const { getLogger, initLogger } = await import("@obs/logger");
initLogger();
const { captureGoldenSetScaffold } = await import("./capture.js");
const logger = getLogger();
const [command = "run", arg1, arg2] = process.argv.slice(2);
const parsed = parseArgs(process.argv.slice(2));
const { command, flags, positionals } = parsed;
if (command === "capture") {
const outputPath = await captureGoldenSetScaffold({
limit: parsePositiveInteger(arg1),
outputPath: arg2,
});
getLogger().info({ outputPath }, '[eval] "capture" completed');
return;
try {
if (command === "capture") {
const { captureGoldenSetScaffold } = await import("./capture.js");
const outputPath = await captureGoldenSetScaffold({
limit:
parsePositiveInteger(getFlagString(flags, "limit")) ??
parsePositiveInteger(positionals[0]),
outputPath: getFlagString(flags, "output") ?? positionals[1],
});
logger.info({ outputPath }, '[eval] "capture" completed');
return;
}
if (command === "run") {
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
"./runner.js"
);
const { publishEvalReportIfPossible } = await import(
"./runner.js"
);
const setName = getFlagString(flags, "set") ?? "golden";
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
if (hasFlag(flags, "dry-run")) {
const cases = await loadEvalCases(inputPath, { limit });
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
return;
}
const { report, reportPath } = await runOnboardingEval({
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
inputPath,
limit,
outputPath: getFlagString(flags, "output"),
setName,
});
const publishedReport = hasFlag(flags, "no-publish-report")
? false
: await publishEvalReportIfPossible(report);
printRunSummary(report, reportPath);
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
return;
}
if (command === "baseline") {
const { updateEvalBaseline } = await import("./runner.js");
const reportPath = getFlagString(flags, "report") ?? positionals[0];
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
console.info(
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
);
return;
}
throw new Error(`Unknown eval command: ${command}`);
} catch (error) {
logger.error(
{ err: error },
`[eval] "${command}" failed`,
);
process.exitCode = 1;
}
printPhaseNotice(command);
}
void main();
+3
View File
@@ -0,0 +1,3 @@
{"id":"greeting-short","name":"Greeting should stay short and welcoming","input":{"request":{"requestId":"eval_greeting_1","skill":"onboarding","chatroomId":"room_eval_1","userId":"user_eval_1","nickname":"<nick:greeting>","trigger":"user-message","userMessage":"안녕하세요"},"session":null,"conversationHistory":[]},"expected":{"intent":"greeting","responseMode":"llm","mustIncludeAny":["환영","반갑"],"mustNotInclude":["디스코드"]}}
{"id":"status-photo-remaining","name":"Status question should answer only the remaining step","input":{"request":{"requestId":"eval_status_1","skill":"onboarding","chatroomId":"room_eval_2","userId":"user_eval_2","nickname":"프리리/95/여/D","trigger":"user-message","userMessage":"이제 뭐 남았지"},"session":{"chatroomId":"room_eval_2","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/D","nicknameVerified":true,"photoVerified":false,"status":"pending","userId":"user_eval_2"},"conversationHistory":[{"role":"user","content":"안녕하세요","timestamp":"2026-04-16T00:00:00.000Z"},{"role":"assistant","content":"환영합니다! 반갑습니다.","timestamp":"2026-04-16T00:00:01.000Z"}]},"expected":{"intent":"status-question","responseMode":"llm","mustIncludeAny":["남았","하트"],"mustNotInclude":["디스코드"]}}
{"id":"nickname-invalid-tier","name":"Nickname invalid flow should focus on nickname correction","input":{"request":{"requestId":"eval_nick_1","skill":"onboarding","chatroomId":"room_eval_3","userId":"user_eval_3","nickname":"프리리/95/여/다이아","trigger":"nickname-invalid"},"session":{"chatroomId":"room_eval_3","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/다이아","nicknameVerified":false,"photoVerified":false,"status":"pending","userId":"user_eval_3"},"conversationHistory":[]},"expected":{"intent":"nickname-question","responseMode":"llm-with-tools","mustIncludeAny":["닉네임","티어","형식"],"mustNotInclude":["디스코드"]}}
+109
View File
@@ -0,0 +1,109 @@
import { describe, expect, it } from "vitest";
import { LlmClient } from "@llm/llm-client";
import type {
ChatCompletionOptions,
ChatCompletionResult,
ChatMessage,
} from "@llm/types";
import { judgeOnboardingWithLlm } from "./llm-judge.js";
import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js";
class FakeLlmClient extends LlmClient {
constructor(private readonly payload: string | null) {
super({ endpoints: [] });
}
hasEndpoints(): boolean {
return this.payload !== null;
}
chatCompletion(
_messages: ChatMessage[],
_options?: ChatCompletionOptions,
): Promise<ChatCompletionResult | null> {
if (this.payload === null) {
return Promise.resolve(null);
}
return Promise.resolve({
content: this.payload,
endpoint: "http://judge.test/v1/chat/completions",
message: {
role: "assistant",
content: this.payload,
},
provider: "cliproxy",
raw: {},
});
}
}
const evalCase: EvalCaseDefinition = {
id: "case-1",
lineNumber: 1,
input: {
request: {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "이제 뭐 남았지",
},
},
expected: {
intent: "status-question",
responseMode: "llm",
},
};
const derived: EvalDerivedSnapshot = {
allowShortResponse: true,
intent: "status-question",
phase: "photo-pending",
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 128,
tier: 0,
};
describe("judgeOnboardingWithLlm", () => {
it("parses a valid JSON judge response", async () => {
const result = await judgeOnboardingWithLlm(
new FakeLlmClient(
JSON.stringify({
friendliness: 5,
accuracy: 4,
concision: 5,
scope: 4,
rationale: "질문 범위에 맞게 짧고 정확하게 답했다.",
}),
),
evalCase,
"이제 방 하트 인증만 남았어요.",
derived,
);
expect(result.skipped).toBe(false);
if (!result.skipped) {
expect(result.scores.overall).toBe(4.5);
expect(result.provider).toBe("cliproxy");
}
});
it("skips when the judge returns invalid JSON", async () => {
const result = await judgeOnboardingWithLlm(
new FakeLlmClient('{"friendliness":"bad"}'),
evalCase,
"이제 방 하트 인증만 남았어요.",
derived,
);
expect(result.skipped).toBe(true);
if (result.skipped) {
expect(result.reason).toBe("invalid-judge-json");
}
});
});
+134
View File
@@ -0,0 +1,134 @@
import { z } from "zod";
import type { LlmClient } from "@llm/llm-client";
import { getLlmJudgeMaxTokens, getLlmJudgeModel } from "../../config.js";
import type {
EvalCaseDefinition,
EvalDerivedSnapshot,
LlmJudgeResult,
LlmJudgeScores,
} from "../types.js";
const llmJudgeSchema = z.object({
friendliness: z.number().int().min(1).max(5),
accuracy: z.number().int().min(1).max(5),
concision: z.number().int().min(1).max(5),
scope: z.number().int().min(1).max(5),
rationale: z.string().trim().min(1).max(200),
});
function round(value: number): number {
return Number(value.toFixed(3));
}
function buildJudgePrompt(
evalCase: EvalCaseDefinition,
responseText: string,
derived: EvalDerivedSnapshot,
): string {
return [
"다음 온보딩 에이전트 응답을 평가해 주세요.",
"평가 축은 friendliness, accuracy, concision, scope 입니다.",
"각 점수는 1~5의 정수만 사용하세요.",
"scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.",
"",
"[request]",
JSON.stringify(evalCase.input.request),
"",
"[expected]",
JSON.stringify({
intent: evalCase.expected.intent ?? null,
mustIncludeAny: evalCase.expected.mustIncludeAny ?? [],
mustNotInclude: evalCase.expected.mustNotInclude ?? [],
responseMode: evalCase.expected.responseMode ?? null,
}),
"",
"[derived]",
JSON.stringify(derived),
"",
"[assistant_response]",
responseText,
"",
'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}',
].join("\n");
}
export async function judgeOnboardingWithLlm(
llmClient: LlmClient,
evalCase: EvalCaseDefinition,
responseText: string,
derived: EvalDerivedSnapshot,
): Promise<LlmJudgeResult> {
if (!responseText.trim()) {
return {
skipped: true,
reason: "empty-response",
};
}
if (!llmClient.hasEndpoints()) {
return {
skipped: true,
reason: "no-llm-endpoint",
};
}
const response = await llmClient.chatCompletion(
[
{
role: "system",
content:
"당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
},
{
role: "user",
content: buildJudgePrompt(evalCase, responseText, derived),
},
],
{
jsonMode: true,
maxTokens: getLlmJudgeMaxTokens(),
modelOverride: getLlmJudgeModel(),
temperature: 0,
},
);
if (!response) {
return {
skipped: true,
reason: "judge-call-failed",
};
}
try {
const parsed = llmJudgeSchema.parse(JSON.parse(response.content));
const scores: LlmJudgeScores = {
friendliness: parsed.friendliness,
accuracy: parsed.accuracy,
concision: parsed.concision,
scope: parsed.scope,
overall: round(
(parsed.friendliness +
parsed.accuracy +
parsed.concision +
parsed.scope) /
4,
),
};
return {
skipped: false,
model: getLlmJudgeModel(),
endpoint: response.endpoint,
provider: response.provider,
scores,
rationale: parsed.rationale,
raw: response.content,
};
} catch {
return {
skipped: true,
reason: "invalid-judge-json",
raw: response.content,
};
}
}
+70
View File
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import { judgeOnboardingRuleCase } from "./rule-judge.js";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
function makeDerived(
overrides?: Partial<DerivedState>,
): DerivedState {
return {
phase: { kind: "photo-pending" },
intent: { kind: "status-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
...overrides,
};
}
describe("judgeOnboardingRuleCase", () => {
it("passes when active checks match", () => {
const result = judgeOnboardingRuleCase(
{
intent: "status-question",
responseMode: "llm",
mustIncludeAny: ["방 하트 인증"],
mustNotInclude: ["디스코드 서버"],
},
"이제 방 하트 인증만 남았어요.",
makeDerived(),
);
expect(result.passed).toBe(true);
expect(result.score).toBe(1);
expect(result.validatorStatus).toBe("pass");
});
it("fails intent and mustNotInclude mismatches", () => {
const result = judgeOnboardingRuleCase(
{
intent: "help-how",
mustNotInclude: ["디스코드"],
},
"디스코드 서버에 먼저 들어오시면 돼요.",
makeDerived(),
);
expect(result.passed).toBe(false);
expect(result.checks.find((check) => check.name === "intent")?.passed).toBe(
false,
);
expect(
result.checks.find((check) => check.name === "mustNotInclude")?.passed,
).toBe(false);
});
it("treats rejecting validator results as a failed rule check", () => {
const result = judgeOnboardingRuleCase(
{},
"디스코드 서버에 들어가시면 돼요.",
makeDerived({
intent: { kind: "photo-question" },
}),
);
expect(result.passed).toBe(false);
expect(result.validatorStatus).toBe("reject");
});
});
+121
View File
@@ -0,0 +1,121 @@
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
function normalizeText(value: string): string {
return value.trim().toLowerCase();
}
function round(value: number): number {
return Number(value.toFixed(3));
}
export function judgeOnboardingRuleCase(
expected: EvalCaseExpected,
responseText: string,
derived: DerivedState,
): RuleJudgeResult {
const validation = validateOnboardingResponse(responseText, derived, {
allowShortResponse: derived.allowShortResponse,
});
const checks: RuleJudgeCheck[] = [];
checks.push({
name: "validator",
passed: validation.status !== "reject",
skipped: false,
detail:
validation.reasons.length > 0
? validation.reasons.join(", ")
: validation.status,
});
const expectedIntent = expected.intent?.trim();
if (expectedIntent) {
checks.push({
name: "intent",
passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind),
skipped: false,
detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`,
});
} else {
checks.push({
name: "intent",
passed: true,
skipped: true,
});
}
const expectedResponseMode = expected.responseMode?.trim();
if (expectedResponseMode) {
checks.push({
name: "responseMode",
passed:
normalizeText(expectedResponseMode) ===
normalizeText(derived.responseMode),
skipped: false,
detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`,
});
} else {
checks.push({
name: "responseMode",
passed: true,
skipped: true,
});
}
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
if (mustIncludeAny.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustIncludeAny.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustIncludeAny",
passed: Boolean(matched),
skipped: false,
detail: matched
? `matched=${matched}`
: `expected one of: ${mustIncludeAny.join(", ")}`,
});
} else {
checks.push({
name: "mustIncludeAny",
passed: true,
skipped: true,
});
}
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
if (mustNotInclude.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustNotInclude.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustNotInclude",
passed: !matched,
skipped: false,
detail: matched ? `forbidden=${matched}` : undefined,
});
} else {
checks.push({
name: "mustNotInclude",
passed: true,
skipped: true,
});
}
const activeChecks = checks.filter((check) => !check.skipped);
const passedChecks = activeChecks.filter((check) => check.passed).length;
return {
passed: activeChecks.every((check) => check.passed),
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
checks,
validatorStatus: validation.status,
validatorReasons: validation.reasons,
};
}
+56
View File
@@ -0,0 +1,56 @@
import { describe, expect, it, vi } from "vitest";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { publishEvalReportIfPossible } from "./runner.js";
import type { EvalReport } from "./types.js";
const natsMocks = vi.hoisted(() => ({
close: vi.fn().mockResolvedValue(undefined),
connect: vi.fn().mockResolvedValue(undefined),
publish: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@common/nats", () => {
return {
NatsClient: class {
connect = natsMocks.connect;
publish = natsMocks.publish;
close = natsMocks.close;
},
};
});
const report: EvalReport = {
generatedAt: "2026-04-17T00:00:00.000Z",
inputPath: "/tmp/onboarding.jsonl",
setName: "golden",
summary: {
averageLatencyMs: 100,
cases: 1,
emptyResponses: 0,
llmAverageAccuracy: 4,
llmAverageConcision: 4,
llmAverageFriendliness: 5,
llmAverageOverall: 4.25,
llmAverageScope: 4,
llmJudgedCases: 1,
passRate: 1,
passedCases: 1,
ruleAverageScore: 1,
},
cases: [],
};
describe("publishEvalReportIfPossible", () => {
it("publishes the report to the eval report subject", async () => {
const result = await publishEvalReportIfPossible(report);
expect(result).toBe(true);
expect(natsMocks.connect).toHaveBeenCalledTimes(1);
expect(natsMocks.publish).toHaveBeenCalledWith(
AGENT_EVAL_REPORT_SUBJECT,
report,
);
expect(natsMocks.close).toHaveBeenCalledTimes(1);
});
});
+842
View File
@@ -0,0 +1,842 @@
import { existsSync } from "node:fs";
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import { basename, dirname, resolve } from "node:path";
import type { AgentRequest } from "@agent/types";
import { GarnetClient } from "@common/garnet";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { NatsClient } from "@common/nats";
import { getConversationSummaryKey } from "@conv/conversation-store";
import { LlmClient } from "@llm/llm-client";
import type { ProviderName } from "@llm/types";
import { runWithContext } from "@obs/context";
import { getLogger } from "@obs/logger";
import type { RoomMessage } from "@room/types";
import { RoomContextService } from "@room/room-context-service";
import { OnboardingSkill } from "@skills/onboarding/index";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import { OnboardingIntentClassifier } from "@skills/onboarding/state/classifier";
import { deriveState } from "@skills/onboarding/state/derive";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import { noopShadowEvalPublisher } from "./shadow-bus.js";
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl, getOpenRouterEnabled } from "../config.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import { judgeOnboardingRuleCase } from "./judges/rule-judge.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalCaseExpected,
EvalCaseInput,
EvalCaseResult,
EvalClassifiedSnapshot,
EvalDerivedSnapshot,
EvalReport,
EvalReportSummary,
EvalSummaryDelta,
} from "./types.js";
const DEFAULT_REPORTS_DIR = "eval-reports";
const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl";
const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl";
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
class InMemoryGarnetClient extends GarnetClient {
private readonly hashStore = new Map<string, Record<string, string>>();
private readonly stringStore = new Map<string, string>();
getJson<T>(key: string): Promise<T | null> {
const raw = this.stringStore.get(key);
return Promise.resolve(raw ? (JSON.parse(raw) as T) : null);
}
setJson(
key: string,
value: unknown,
_ttlSeconds?: number,
): Promise<void> {
this.stringStore.set(key, JSON.stringify(value));
return Promise.resolve();
}
getString(key: string): Promise<string | null> {
return Promise.resolve(this.stringStore.get(key) ?? null);
}
setString(
key: string,
value: string,
_ttlSeconds?: number,
): Promise<void> {
this.stringStore.set(key, value);
return Promise.resolve();
}
delete(key: string): Promise<number> {
const stringDeleted = this.stringStore.delete(key);
const hashDeleted = this.hashStore.delete(key);
return Promise.resolve(stringDeleted || hashDeleted ? 1 : 0);
}
getHash(key: string): Promise<Record<string, string>> {
return Promise.resolve({ ...(this.hashStore.get(key) ?? {}) });
}
setHash(
key: string,
fields: Record<string, string>,
): Promise<void> {
this.hashStore.set(key, { ...fields });
return Promise.resolve();
}
}
class InMemoryRoomContextService extends RoomContextService {
constructor(private readonly messages: RoomMessage[]) {
super();
}
getRecentMessages(
_chatroomId: string,
limit?: number,
): Promise<RoomMessage[]> {
const effectiveLimit = limit ?? this.messages.length;
return Promise.resolve(this.messages.slice(-effectiveLimit));
}
}
interface RawEvalCaseLine {
expected?: Record<string, unknown>;
id?: unknown;
input?: Record<string, unknown>;
name?: unknown;
notes?: unknown;
}
export interface EvalRunOptions {
enableLlmJudge?: boolean | undefined;
inputPath?: string | undefined;
limit?: number | undefined;
outputPath?: string | undefined;
setName?: string | undefined;
}
function round(value: number): number {
return Number(value.toFixed(3));
}
function average(values: number[]): number | null {
if (values.length === 0) {
return null;
}
return round(values.reduce((sum, value) => sum + value, 0) / values.length);
}
function averageOrZero(values: number[]): number {
return values.length === 0 ? 0 : round(values.reduce((sum, value) => sum + value, 0) / values.length);
}
function firstNonEmpty(
...values: Array<string | null | undefined>
): string | undefined {
for (const value of values) {
if (value?.trim()) {
return value.trim();
}
}
return undefined;
}
function parseEndpointList(
...values: Array<string | null | undefined>
): string[] {
const endpoints: string[] = [];
for (const value of values) {
if (!value?.trim()) {
continue;
}
for (const item of value.split(/[\n,]/u)) {
const trimmed = item.trim();
if (trimmed) {
endpoints.push(trimmed);
}
}
}
return [...new Set(endpoints)];
}
function toProviderEndpoints(
urls: string[],
provider?: ProviderName,
): Array<string | { url: string; provider?: ProviderName | undefined }> {
if (!provider) {
return urls;
}
return urls.map((url) => ({ url, provider }));
}
function createEvalLlmClient(): LlmClient {
const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]);
const llamaEndpoints = parseEndpointList(
process.env["AGENT_LLM_API_URLS"],
process.env["LLAMA_SERVER_API_URL"],
);
const openRouterEndpoints = getOpenRouterEnabled()
? parseEndpointList(process.env["OPENROUTER_API_URL"], getOpenRouterApiUrl())
: [];
const llmApiKey = firstNonEmpty(
process.env["AGENT_LLM_API_KEY"],
process.env["LLAMA_SERVER_API_KEY"],
);
const providerApiKeys = Object.fromEntries(
[
["cliproxy", firstNonEmpty(process.env["CLI_PROXY_API_KEY"])],
["llama", llmApiKey],
["openrouter", firstNonEmpty(process.env["OPENROUTER_API_KEY"])],
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
) as Partial<Record<ProviderName, string>>;
return new LlmClient({
endpoints: [
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
...toProviderEndpoints(llamaEndpoints, "llama"),
],
apiKey: llmApiKey,
model: firstNonEmpty(process.env["AGENT_LLM_MODEL"]),
providerApiKeys,
});
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
function getStringField(record: Record<string, unknown>, key: string): string | undefined {
const value = record[key];
return typeof value === "string" ? value : undefined;
}
function getStringArrayField(
record: Record<string, unknown>,
key: string,
): string[] | undefined {
const value = record[key];
if (!Array.isArray(value)) {
return undefined;
}
const strings = value.filter((item): item is string => typeof item === "string");
return strings.length > 0 ? strings : [];
}
function toAgentRequest(raw: Record<string, unknown>): AgentRequest {
const requestId = getStringField(raw, "requestId");
const skill = getStringField(raw, "skill");
const chatroomId = getStringField(raw, "chatroomId");
const userId = getStringField(raw, "userId");
const nickname = getStringField(raw, "nickname");
const trigger = getStringField(raw, "trigger");
if (!requestId || !skill || !chatroomId || !userId || !nickname || !trigger) {
throw new Error("input.request is missing required AgentRequest fields");
}
const request: AgentRequest = {
requestId,
skill,
chatroomId,
userId,
nickname,
trigger,
};
const userMessage = getStringField(raw, "userMessage");
if (userMessage !== undefined) {
request.userMessage = userMessage;
}
const context = raw["context"];
if (isRecord(context)) {
request.context = context;
}
return request;
}
function parseConversationHistory(raw: unknown): EvalCaseInput["conversationHistory"] {
if (!Array.isArray(raw)) {
return undefined;
}
return raw
.filter((item): item is Record<string, unknown> => isRecord(item))
.map((item) => {
const role = item["role"];
const content = getStringField(item, "content") ?? "";
const timestamp = getStringField(item, "timestamp") ?? new Date(0).toISOString();
if (role !== "user" && role !== "assistant") {
throw new Error("conversationHistory contains an invalid role");
}
return {
role,
content,
timestamp,
};
});
}
function parseRoomContext(raw: unknown): EvalCaseInput["roomContext"] {
if (!Array.isArray(raw)) {
return undefined;
}
return raw
.filter((item): item is Record<string, unknown> => isRecord(item))
.map((item) => ({
senderId: getStringField(item, "senderId") ?? "",
nickname: getStringField(item, "nickname") ?? "알 수 없음",
message: getStringField(item, "message") ?? "",
timestamp: getStringField(item, "timestamp") ?? new Date(0).toISOString(),
}));
}
function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpected {
if (!raw) {
return {};
}
const expected: EvalCaseExpected = {};
const intent = getStringField(raw, "intent");
const responseMode = getStringField(raw, "responseMode");
const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny");
const mustNotInclude = getStringArrayField(raw, "mustNotInclude");
if (intent) {
expected.intent = intent;
}
if (responseMode) {
expected.responseMode = responseMode;
}
if (mustIncludeAny) {
expected.mustIncludeAny = mustIncludeAny;
}
if (mustNotInclude) {
expected.mustNotInclude = mustNotInclude;
}
return expected;
}
function parseEvalCaseLine(
raw: RawEvalCaseLine,
lineNumber: number,
sourcePath: string,
): EvalCaseDefinition {
if (!isRecord(raw.input)) {
throw new Error("input is required");
}
const requestRaw = raw.input["request"];
if (!isRecord(requestRaw)) {
throw new Error("input.request is required");
}
const input: EvalCaseInput = {
request: toAgentRequest(requestRaw),
};
const conversationHistory = parseConversationHistory(raw.input["conversationHistory"]);
if (conversationHistory) {
input.conversationHistory = conversationHistory;
}
const roomContext = parseRoomContext(raw.input["roomContext"]);
if (roomContext) {
input.roomContext = roomContext;
}
if (raw.input["summary"] !== undefined) {
const summary = getStringField(raw.input, "summary");
if (summary !== undefined) {
input.summary = summary;
}
}
if (raw.input["session"] === null || isRecord(raw.input["session"])) {
input.session = (raw.input["session"] ?? null) as OnboardingSession | null;
}
if (raw.input["blackboard"] === null || isRecord(raw.input["blackboard"])) {
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
}
return {
id:
typeof raw.id === "string" && raw.id.trim()
? raw.id.trim()
: `${basename(sourcePath)}:${lineNumber}`,
lineNumber,
name: typeof raw.name === "string" ? raw.name : undefined,
notes: typeof raw.notes === "string" ? raw.notes : undefined,
input,
expected: parseExpected(isRecord(raw.expected) ? raw.expected : undefined),
};
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
return {
allowShortResponse: derived.allowShortResponse,
intent: derived.intent.kind,
phase: derived.phase.kind,
responseMode: derived.responseMode,
shouldRespond: derived.shouldRespond,
suggestedMaxTokens: derived.suggestedMaxTokens,
tier: derived.tier,
};
}
function toClassifiedSnapshot(classified: ClassifiedIntent): EvalClassifiedSnapshot {
return {
confidence: round(classified.confidence),
intent: classified.intent.kind,
source: classified.source,
};
}
function toRoomMessages(
input: EvalCaseInput["roomContext"],
): RoomMessage[] {
return (
input?.map((message) => ({
senderId: message.senderId,
nickname: message.nickname,
message: message.message,
timestamp: new Date(message.timestamp),
})) ?? []
);
}
async function hydrateEvalState(
garnetClient: InMemoryGarnetClient,
input: EvalCaseInput,
): Promise<void> {
const {
request: { chatroomId, userId },
} = input;
if (input.session !== undefined) {
await garnetClient.setJson(
`${getOnboardingSessionKeyPrefix()}:onboarding:${chatroomId}:${userId}`,
input.session,
);
}
if (input.blackboard !== undefined) {
await garnetClient.setJson(
`lolympus-agent:bb:${chatroomId}:${userId}`,
input.blackboard,
);
}
if (input.summary?.trim()) {
await garnetClient.setString(
getConversationSummaryKey(chatroomId, userId),
input.summary,
);
}
}
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
const llmResults = results.flatMap((result) =>
result.llmJudge.skipped ? [] : [result.llmJudge],
);
return {
averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)),
cases: results.length,
emptyResponses: results.filter((result) => !result.response.text.trim()).length,
llmAverageAccuracy: average(
llmResults.map((result) => result.scores.accuracy),
),
llmAverageConcision: average(
llmResults.map((result) => result.scores.concision),
),
llmAverageFriendliness: average(
llmResults.map((result) => result.scores.friendliness),
),
llmAverageOverall: average(
llmResults.map((result) => result.scores.overall),
),
llmAverageScope: average(
llmResults.map((result) => result.scores.scope),
),
llmJudgedCases: llmResults.length,
passRate:
results.length === 0
? 0
: round(
results.filter((result) => result.ruleJudge.passed).length /
results.length,
),
passedCases: results.filter((result) => result.ruleJudge.passed).length,
ruleAverageScore: averageOrZero(
results.map((result) => result.ruleJudge.score),
),
};
}
function createBaselineDelta(
current: EvalReportSummary,
baseline: EvalReportSummary,
): EvalSummaryDelta {
const subtract = (
left: number | null,
right: number | null,
): number | null => {
if (left === null || right === null) {
return null;
}
return round(left - right);
};
return {
averageLatencyMs: subtract(current.averageLatencyMs, baseline.averageLatencyMs),
llmAverageOverall: subtract(
current.llmAverageOverall,
baseline.llmAverageOverall,
),
passRate: subtract(current.passRate, baseline.passRate),
ruleAverageScore: subtract(current.ruleAverageScore, baseline.ruleAverageScore),
};
}
function buildDefaultReportPath(timestamp: string): string {
const filename = `${timestamp.replace(/[:.]/gu, "-")}.json`;
return resolve(process.cwd(), DEFAULT_REPORTS_DIR, filename);
}
async function writeReport(
report: EvalReport,
outputPath: string,
): Promise<string> {
const resolved = resolve(process.cwd(), outputPath);
await mkdir(dirname(resolved), { recursive: true });
await writeFile(resolved, `${JSON.stringify(report, null, 2)}\n`, "utf8");
return resolved;
}
async function loadBaselineFile(): Promise<EvalBaselineFile | null> {
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
if (!existsSync(baselinePath)) {
return null;
}
const raw = await readFile(baselinePath, "utf8");
return JSON.parse(raw) as EvalBaselineFile;
}
export function resolveEvalSetPath(setNameOrPath = "golden"): string {
if (setNameOrPath === "golden") {
return resolve(process.cwd(), GOLDEN_SET_PATH);
}
if (setNameOrPath === "capture") {
return resolve(process.cwd(), CAPTURE_SET_PATH);
}
if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) {
return resolve(process.cwd(), setNameOrPath);
}
return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`);
}
export async function loadEvalCases(
inputPath: string,
options?: { limit?: number | undefined },
): Promise<EvalCaseDefinition[]> {
const resolvedPath = resolve(process.cwd(), inputPath);
if (!existsSync(resolvedPath)) {
throw new Error(`Eval set not found: ${resolvedPath}`);
}
const raw = await readFile(resolvedPath, "utf8");
const lines = raw.split(/\r?\n/u).filter((line) => line.trim());
const cases = lines.map((line, index) =>
parseEvalCaseLine(
JSON.parse(line) as RawEvalCaseLine,
index + 1,
resolvedPath,
),
);
return options?.limit ? cases.slice(0, options.limit) : cases;
}
async function evaluateCase(
evalCase: EvalCaseDefinition,
llmClient: LlmClient,
enableLlmJudge: boolean,
): Promise<EvalCaseResult> {
const input = evalCase.input;
const inMemoryGarnet = new InMemoryGarnetClient();
await hydrateEvalState(inMemoryGarnet, input);
const roomContextService =
input.roomContext && input.roomContext.length > 0
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
: undefined;
const skill = new OnboardingSkill(
inMemoryGarnet,
llmClient,
roomContextService,
noopShadowEvalPublisher,
);
const classifier = new OnboardingIntentClassifier(llmClient);
return runWithContext(
{
chatroomId: input.request.chatroomId,
deadlineAt: new Date(Date.now() + 4_500).toISOString(),
requestId: input.request.requestId,
skill: input.request.skill,
startedAt: new Date().toISOString(),
trigger: input.request.trigger,
userId: input.request.userId,
},
async () => {
const classified = await classifier.classify(
input.request.userMessage,
input.request.trigger,
);
const derived = deriveState(
input.session ?? null,
input.request.trigger,
classified.intent,
);
const startedAt = Date.now();
try {
const response = await skill.handle(
input.request,
input.conversationHistory ?? [],
);
const latencyMs = Date.now() - startedAt;
const derivedSnapshot = toDerivedSnapshot(derived);
const ruleJudge = judgeOnboardingRuleCase(
evalCase.expected,
response.text,
derived,
);
const llmJudge = enableLlmJudge
? await judgeOnboardingWithLlm(
llmClient,
evalCase,
response.text,
derivedSnapshot,
)
: {
skipped: true as const,
reason: "disabled",
};
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: derivedSnapshot,
request: input.request,
response,
ruleJudge,
llmJudge,
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
const errorMessage =
error instanceof Error ? error.message : String(error);
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: toDerivedSnapshot(derived),
request: input.request,
response: {
requestId: input.request.requestId,
text: "",
confidence: 0,
source: "template",
metadata: {
error: errorMessage,
},
},
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
llmJudge: {
skipped: true,
reason: "skill-error",
},
error: errorMessage,
};
}
},
);
}
export async function runOnboardingEval(
options?: EvalRunOptions,
): Promise<{ report: EvalReport; reportPath: string }> {
const logger = getLogger();
const setName = options?.setName ?? "golden";
const inputPath = options?.inputPath
? resolve(process.cwd(), options.inputPath)
: resolveEvalSetPath(setName);
const evalCases = await loadEvalCases(inputPath, {
limit: options?.limit,
});
const llmClient = createEvalLlmClient();
const enableLlmJudge = options?.enableLlmJudge ?? true;
const caseResults: EvalCaseResult[] = [];
for (const evalCase of evalCases) {
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge));
}
const summary = createSummary(caseResults);
const baseline = await loadBaselineFile();
const report: EvalReport = {
generatedAt: new Date().toISOString(),
inputPath,
setName,
summary,
cases: caseResults,
};
if (baseline) {
report.baselineDelta = createBaselineDelta(summary, baseline.summary);
}
const reportPath = await writeReport(
report,
options?.outputPath ?? buildDefaultReportPath(report.generatedAt),
);
logger.info(
{
cases: summary.cases,
passRate: summary.passRate,
reportPath,
},
"[Eval] run completed",
);
return { report, reportPath };
}
export async function findLatestEvalReportPath(): Promise<string | null> {
const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR);
if (!existsSync(reportsDir)) {
return null;
}
const entries = await readdir(reportsDir, { withFileTypes: true });
const candidates = entries
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
.filter((name) => name.endsWith(".json") && name !== "baseline.json")
.sort((left, right) => right.localeCompare(left));
if (candidates.length === 0) {
return null;
}
const latest = candidates[0];
if (!latest) {
return null;
}
return resolve(reportsDir, latest);
}
export async function updateEvalBaseline(
reportPath?: string,
): Promise<{
baseline: EvalBaselineFile;
baselinePath: string;
storedInRedis: boolean;
}> {
const sourcePath =
reportPath?.trim() ? resolve(process.cwd(), reportPath) : await findLatestEvalReportPath();
if (!sourcePath) {
throw new Error("No eval report found to baseline");
}
const raw = await readFile(sourcePath, "utf8");
const report = JSON.parse(raw) as EvalReport;
const baseline: EvalBaselineFile = {
reportPath: sourcePath,
setName: report.setName,
summary: report.summary,
updatedAt: new Date().toISOString(),
};
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
await mkdir(dirname(baselinePath), { recursive: true });
await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
const storedInRedis = await persistBaselineInRedis(baseline);
return { baseline, baselinePath, storedInRedis };
}
async function persistBaselineInRedis(
baseline: EvalBaselineFile,
): Promise<boolean> {
try {
const { GarnetClient } = await import("@common/garnet");
const garnetClient = new GarnetClient();
await garnetClient.connect();
try {
await garnetClient.setString(
EVAL_BASELINE_KEY,
JSON.stringify(baseline),
);
return true;
} finally {
await garnetClient.close();
}
} catch {
return false;
}
}
export async function publishEvalReportIfPossible(
report: EvalReport,
): Promise<boolean> {
try {
const natsClient = new NatsClient();
await natsClient.connect();
try {
await natsClient.publish(AGENT_EVAL_REPORT_SUBJECT, report);
return true;
} finally {
await natsClient.close();
}
} catch {
return false;
}
}
+93
View File
@@ -0,0 +1,93 @@
import { describe, expect, it, vi } from "vitest";
import type { AgentRequest, AgentResponse } from "@agent/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
import {
ShadowEvalBus,
shouldSampleShadowEval,
} from "./shadow-bus.js";
const request: AgentRequest = {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "이제 뭐 남았지",
};
const response: AgentResponse = {
requestId: "req-1",
text: "이제 방 하트 인증만 남았어요.",
confidence: 0.9,
source: "llm",
};
const classified: ClassifiedIntent = {
intent: { kind: "status-question" },
confidence: 0.98,
source: "regex",
};
const derived: DerivedState = {
phase: { kind: "photo-pending" },
intent: { kind: "status-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
};
describe("shouldSampleShadowEval", () => {
it("samples when the random value is under the tier rate", () => {
expect(shouldSampleShadowEval(0, 0)).toBe(true);
});
it("does not sample when the random value exceeds the tier rate", () => {
expect(shouldSampleShadowEval(0, 1)).toBe(false);
});
});
describe("ShadowEvalBus", () => {
it("publishes a shadow-eval stream entry when sampled", () => {
const appendStream = vi.fn().mockResolvedValue("1-0");
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
new ShadowEvalBus({ appendStream }).publish({
classified,
createdAt: "2026-04-16T00:00:00.000Z",
derived,
request,
response,
});
expect(appendStream).toHaveBeenCalledTimes(1);
expect(appendStream.mock.calls[0]?.[0]).toBe("lolympus-agent:background");
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
type: "shadow-eval",
requestId: "req-1",
tier: "0",
});
randomSpy.mockRestore();
});
it("skips publishing for empty responses", () => {
const appendStream = vi.fn().mockResolvedValue("1-0");
new ShadowEvalBus({ appendStream }).publish({
classified,
createdAt: "2026-04-16T00:00:00.000Z",
derived,
request,
response: {
...response,
text: "",
},
});
expect(appendStream).not.toHaveBeenCalled();
});
});
+90
View File
@@ -0,0 +1,90 @@
import type { AgentRequest, AgentResponse } from "@agent/types";
import type { GarnetClient } from "@common/garnet";
import { getRequestLogger } from "@obs/context";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import {
getBackgroundStreamMaxLen,
getShadowEvalSamplingRate,
} from "../config.js";
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
export interface ShadowEvalEnvelope {
classified: ClassifiedIntent;
createdAt: string;
derived: DerivedState;
latencyMs?: number | undefined;
request: AgentRequest;
response: AgentResponse;
}
export interface ShadowEvalPublisher {
publish(envelope: ShadowEvalEnvelope): void;
}
export const noopShadowEvalPublisher: ShadowEvalPublisher = {
publish(): void {},
};
export function shouldSampleShadowEval(
tier: 0 | 1 | 2,
randomValue = Math.random(),
): boolean {
return randomValue < getShadowEvalSamplingRate(tier);
}
export class ShadowEvalBus implements ShadowEvalPublisher {
constructor(
private readonly garnetClient: Pick<GarnetClient, "appendStream">,
) {}
publish(envelope: ShadowEvalEnvelope): void {
if (!envelope.response.text.trim()) {
return;
}
if (!shouldSampleShadowEval(envelope.derived.tier)) {
return;
}
const logger = getRequestLogger();
void this.garnetClient
.appendStream(
BACKGROUND_STREAM_KEY,
{
type: "shadow-eval",
requestId: envelope.request.requestId,
chatroomId: envelope.request.chatroomId,
userId: envelope.request.userId,
tier: String(envelope.derived.tier),
latencyMs:
envelope.latencyMs !== undefined
? String(envelope.latencyMs)
: "",
input: JSON.stringify({
request: envelope.request,
}),
output: JSON.stringify(envelope.response),
derived: JSON.stringify(envelope.derived),
classified: JSON.stringify(envelope.classified),
createdAt: envelope.createdAt,
},
{
maxLen: getBackgroundStreamMaxLen(),
},
)
.catch((error) => {
logger.warn(
{
err: error,
requestId: envelope.request.requestId,
},
"[ShadowEvalBus] Failed to publish shadow eval job",
);
});
}
}
+53
View File
@@ -0,0 +1,53 @@
import { describe, expect, it } from "vitest";
import {
calculateEvalScoreDropPct,
calculateLlmErrorRate,
calculatePercentile,
} from "./shadow-worker.js";
describe("calculatePercentile", () => {
it("returns null for empty input", () => {
expect(calculatePercentile([], 0.95)).toBeNull();
});
it("calculates the p95 from sorted values", () => {
expect(calculatePercentile([100, 200, 300, 400, 500], 0.95)).toBe(500);
});
});
describe("calculateEvalScoreDropPct", () => {
it("returns null without a valid baseline", () => {
expect(calculateEvalScoreDropPct(4.2, null)).toBeNull();
expect(calculateEvalScoreDropPct(4.2, 0)).toBeNull();
});
it("computes percentage drop from baseline", () => {
expect(calculateEvalScoreDropPct(3.6, 4.0)).toBeCloseTo(10);
});
});
describe("calculateLlmErrorRate", () => {
it("returns null when no observations are present", () => {
expect(calculateLlmErrorRate([])).toBeNull();
});
it("computes error ratio across observations", () => {
expect(
calculateLlmErrorRate([
{
llmJudgeErrored: true,
llmOverall: null,
recordedAt: 1,
responseLatencyMs: 100,
},
{
llmJudgeErrored: false,
llmOverall: 4.2,
recordedAt: 2,
responseLatencyMs: 200,
},
]),
).toBe(0.5);
});
});
+741
View File
@@ -0,0 +1,741 @@
import { existsSync } from "node:fs";
import { readFile } from "node:fs/promises";
import { resolve } from "node:path";
import type { AgentRequest, AgentResponse } from "@agent/types";
import type {
GarnetClient,
StreamEntry,
StreamPendingEntry,
} from "@common/garnet";
import { summarizeConversation } from "@conv/summarizer";
import type { LlmClient } from "@llm/llm-client";
import { getLogger } from "@obs/logger";
import type { AlertSender } from "@obs/alert";
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ConversationStore } from "@conv/conversation-store";
import {
getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs,
getEvalStreamMaxLen,
} from "../config.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalDerivedSnapshot,
} from "./types.js";
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
const BACKGROUND_GROUP = "bg-worker";
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
const ALERT_WINDOW_MS = 5 * 60_000;
const ALERT_WINDOW_LABEL = "5m";
const ALERT_TICK_MS = 60_000;
const MAX_DELIVERIES = 3;
const MIN_ERROR_RATE_OBSERVATIONS = 5;
const MIN_LATENCY_OBSERVATIONS = 5;
const MIN_SCORE_OBSERVATIONS = 3;
const PENDING_IDLE_MS = 1_000;
const READ_BLOCK_MS = 5_000;
const READ_COUNT = 10;
const PENDING_BATCH_SIZE = 20;
const STOP_TIMEOUT_MS = 10_000;
type BackgroundJobType = "shadow-eval" | "summarize";
interface ShadowEvalJob {
classified: ClassifiedIntent;
createdAt: string;
derived: DerivedState;
latencyMs?: number | undefined;
request: AgentRequest;
response: AgentResponse;
}
interface EvalObservation {
llmJudgeErrored: boolean;
llmOverall: number | null;
recordedAt: number;
responseLatencyMs: number | null;
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
function promiseWithTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
): Promise<T> {
return Promise.race([
promise,
new Promise<T>((_, reject) => {
setTimeout(() => {
reject(new Error(`Timed out after ${timeoutMs}ms`));
}, timeoutMs);
}),
]);
}
function getJobType(message: StreamEntry): string {
return message.message["type"] ?? "unknown";
}
function parseOptionalNumberField(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : undefined;
}
function parseJsonField<T>(
value: string | undefined,
fieldName: string,
): T {
if (!value?.trim()) {
throw new Error(`Missing ${fieldName} field`);
}
return JSON.parse(value) as T;
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
return {
allowShortResponse: derived.allowShortResponse,
intent: derived.intent.kind,
phase: derived.phase.kind,
responseMode: derived.responseMode,
shouldRespond: derived.shouldRespond,
suggestedMaxTokens: derived.suggestedMaxTokens,
tier: derived.tier,
};
}
function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition {
return {
id: `shadow:${messageId}`,
lineNumber: 0,
input: {
request: job.request,
},
expected: {},
};
}
function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
const input = parseJsonField<{ request: AgentRequest }>(
message.message["input"],
"input",
);
return {
classified: parseJsonField<ClassifiedIntent>(
message.message["classified"],
"classified",
),
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
derived: parseJsonField<DerivedState>(
message.message["derived"],
"derived",
),
latencyMs: parseOptionalNumberField(message.message["latencyMs"]),
request: input.request,
response: parseJsonField<AgentResponse>(
message.message["output"],
"output",
),
};
}
export function calculatePercentile(
values: number[],
percentile: number,
): number | null {
if (values.length === 0) {
return null;
}
const sorted = [...values].sort((left, right) => left - right);
const index = Math.min(
sorted.length - 1,
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
);
const selected = sorted[index];
return selected === undefined ? null : selected;
}
export function calculateEvalScoreDropPct(
currentAverage: number | null,
baselineAverage: number | null,
): number | null {
if (
currentAverage === null ||
baselineAverage === null ||
baselineAverage <= 0
) {
return null;
}
return ((baselineAverage - currentAverage) / baselineAverage) * 100;
}
export function calculateLlmErrorRate(
observations: EvalObservation[],
): number | null {
if (observations.length === 0) {
return null;
}
const errorCount = observations.filter(
(observation) => observation.llmJudgeErrored,
).length;
return errorCount / observations.length;
}
export class ShadowWorker {
private lastAggregateAt = 0;
private loopPromise: Promise<void> | null = null;
private readonly logger = getLogger();
private readonly recentEvalObservations: EvalObservation[] = [];
private stopRequested = false;
constructor(
private readonly garnetClient: GarnetClient,
private readonly llmClient: LlmClient,
private readonly conversationStore: ConversationStore,
private readonly alertSender?: AlertSender,
private readonly consumerName = `bg-worker-${process.pid}`,
) {}
async start(): Promise<void> {
await this.ensureConsumerGroup();
if (this.loopPromise) {
return;
}
this.stopRequested = false;
this.loopPromise = this.runLoop();
}
async stop(): Promise<void> {
this.stopRequested = true;
if (!this.loopPromise) {
return;
}
try {
await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS);
} catch (error) {
this.logger.warn(
{ err: error },
"[ShadowWorker] Stop timed out while waiting for the loop to finish",
);
} finally {
this.loopPromise = null;
}
}
private async ensureConsumerGroup(): Promise<void> {
try {
await this.garnetClient.createConsumerGroup(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
);
} catch (error) {
if (!String(error).includes("BUSYGROUP")) {
throw error;
}
}
}
private async runLoop(): Promise<void> {
while (!this.stopRequested) {
try {
await this.runPeriodicChecksIfNeeded();
const recoveredPending = await this.recoverPendingOnce();
if (recoveredPending) {
continue;
}
const streams = await this.garnetClient.readGroup(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
this.consumerName,
">",
{
blockMs: READ_BLOCK_MS,
count: READ_COUNT,
},
);
if (!streams) {
continue;
}
for (const stream of streams) {
for (const message of stream.messages) {
if (this.stopRequested) {
return;
}
await this.processMessage(
message,
message.deliveriesCounter ?? 1,
);
}
}
} catch (error) {
this.logger.error(
{ err: error },
"[ShadowWorker] Loop iteration failed",
);
await delay(1_000);
}
}
}
private async recoverPendingOnce(): Promise<boolean> {
const pending = await this.garnetClient.getPendingStreamRange(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
PENDING_BATCH_SIZE,
);
const recoverable = pending.filter(
(entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS,
);
if (recoverable.length === 0) {
return false;
}
for (const entry of recoverable) {
if (this.stopRequested) {
break;
}
await this.recoverPendingEntry(entry);
}
return true;
}
private async recoverPendingEntry(
entry: StreamPendingEntry,
): Promise<void> {
const nextDeliveryCount = entry.deliveriesCounter + 1;
const claimed = await this.garnetClient.claimPendingStream(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
this.consumerName,
PENDING_IDLE_MS,
entry.id,
{
retryCount: nextDeliveryCount,
},
);
const message = claimed[0];
if (!message) {
return;
}
if (nextDeliveryCount > MAX_DELIVERIES) {
await this.handleMaxDeliveries(
message,
nextDeliveryCount,
"recovered entry exceeded max deliveries",
);
return;
}
await this.processMessage(message, nextDeliveryCount);
}
private async processMessage(
message: StreamEntry,
deliveryCount: number,
): Promise<void> {
try {
switch (getJobType(message) as BackgroundJobType) {
case "shadow-eval":
await this.processShadowEval(message);
await this.acknowledge(message.id);
return;
case "summarize":
await this.processSummarize(message);
await this.acknowledge(message.id);
return;
default:
this.logger.warn(
{
messageId: message.id,
type: getJobType(message),
},
"[ShadowWorker] Unknown background job type; acknowledging",
);
await this.acknowledge(message.id);
}
} catch (error) {
this.logger.warn(
{
err: error,
deliveryCount,
messageId: message.id,
type: getJobType(message),
},
"[ShadowWorker] Job processing failed",
);
if (deliveryCount >= MAX_DELIVERIES) {
await this.handleMaxDeliveries(
message,
deliveryCount,
error instanceof Error ? error.message : String(error),
);
}
}
}
private async processShadowEval(message: StreamEntry): Promise<void> {
const job = parseShadowEvalJob(message);
const validation = validateOnboardingResponse(
job.response.text,
job.derived,
{
allowShortResponse: job.derived.allowShortResponse,
},
);
const llmJudge = await judgeOnboardingWithLlm(
this.llmClient,
toEvalCase(job, message.id),
job.response.text,
toDerivedSnapshot(job.derived),
);
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
const llmJudgeErrored =
llmJudge.skipped &&
llmJudge.reason !== "empty-response" &&
llmJudge.reason !== "disabled";
this.recordEvalObservation({
llmJudgeErrored,
llmOverall,
recordedAt: Date.now(),
responseLatencyMs: job.latencyMs ?? null,
});
await this.garnetClient.appendStream(
EVAL_RESULTS_STREAM_KEY,
{
type: "shadow-eval-result",
requestId: job.request.requestId,
chatroomId: job.request.chatroomId,
userId: job.request.userId,
tier: String(job.derived.tier),
intent: job.derived.intent.kind,
responseMode: job.derived.responseMode,
validatorStatus: validation.status,
latencyMs:
job.latencyMs !== undefined ? String(job.latencyMs) : "",
llmOverall:
llmOverall === null ? "" : String(llmOverall),
input: JSON.stringify({
request: job.request,
}),
output: JSON.stringify(job.response),
derived: JSON.stringify(job.derived),
classified: JSON.stringify(job.classified),
validation: JSON.stringify(validation),
llmJudge: JSON.stringify(llmJudge),
evaluatedAt: new Date().toISOString(),
},
{
maxLen: getEvalStreamMaxLen(),
},
);
}
private async processSummarize(message: StreamEntry): Promise<void> {
const chatroomId = message.message["chatroomId"];
const userId = message.message["userId"];
if (!chatroomId?.trim() || !userId?.trim()) {
throw new Error("Missing summarize job identifiers");
}
const [history, existingSummary] = await Promise.all([
this.conversationStore.getHistory(chatroomId, userId),
this.conversationStore.getSummary(chatroomId, userId),
]);
if (history.length === 0) {
return;
}
const summary = await summarizeConversation(
this.llmClient,
history,
existingSummary ?? undefined,
);
await this.conversationStore.setSummary(chatroomId, userId, summary);
}
private async handleMaxDeliveries(
message: StreamEntry,
deliveryCount: number,
lastError: string,
): Promise<void> {
const type = getJobType(message);
if (type === "summarize") {
this.logger.warn(
{
deliveryCount,
lastError,
messageId: message.id,
},
"[ShadowWorker] Dropping summarize job after max retries",
);
await this.acknowledge(message.id);
return;
}
await this.garnetClient.appendStream(
BACKGROUND_DLQ_STREAM_KEY,
{
type,
payload: JSON.stringify(message.message),
retryCount: String(deliveryCount),
lastError: lastError.slice(0, 500),
originalId: message.id,
},
{
maxLen: 1_000,
},
);
await this.acknowledge(message.id);
}
private acknowledge(id: string): Promise<number> {
return this.garnetClient.acknowledgeStream(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
id,
);
}
private recordEvalObservation(observation: EvalObservation): void {
this.recentEvalObservations.push(observation);
this.pruneOldObservations(Date.now());
}
private pruneOldObservations(now: number): void {
const cutoff = now - ALERT_WINDOW_MS;
let removeCount = 0;
for (const observation of this.recentEvalObservations) {
if (observation.recordedAt >= cutoff) {
break;
}
removeCount += 1;
}
if (removeCount > 0) {
this.recentEvalObservations.splice(0, removeCount);
}
}
private async runPeriodicChecksIfNeeded(): Promise<void> {
if (!this.alertSender) {
return;
}
const now = Date.now();
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
return;
}
this.lastAggregateAt = now;
this.pruneOldObservations(now);
await this.checkLatencyAlert(now);
await this.checkEvalScoreDropAlert(now);
await this.checkLlmErrorRateAlert(now);
await this.checkOpenCircuitAlerts(now);
}
private async checkLatencyAlert(now: number): Promise<void> {
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
);
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
return;
}
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
const threshold = getAlertThresholdP95LatencyMs();
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: p95LatencyMs,
window: ALERT_WINDOW_LABEL,
});
}
private async checkEvalScoreDropAlert(now: number): Promise<void> {
if (!this.alertSender) {
return;
}
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
observation.llmOverall === null ? [] : [observation.llmOverall],
);
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
return;
}
const currentAverage =
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
const baselineAverage = await this.loadBaselineOverallScore();
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
const threshold = getAlertThresholdEvalScoreDropPct();
if (dropPct === null || dropPct <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "eval-score-drop",
metric: "shadow_eval.llm_overall_drop_pct",
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(dropPct.toFixed(3)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkLlmErrorRateAlert(now: number): Promise<void> {
if (
!this.alertSender ||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
) {
return;
}
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
const threshold = getAlertThresholdLlmErrorRate();
if (errorRate === null || errorRate <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "llm-error-rate",
metric: "shadow_eval.llm_error_rate",
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(errorRate.toFixed(4)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkOpenCircuitAlerts(now: number): Promise<void> {
if (!this.alertSender) {
return;
}
const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*");
for (const key of circuitKeys) {
const record = await this.garnetClient.getHash(key);
if (record["state"] !== "open") {
continue;
}
const endpointHash = key.split(":").at(-1) ?? "unknown";
const failureCount = Number(record["failureCount"]);
await this.alertSender.sendAlert(
{
kind: "endpoint-down",
metric: "llm.circuit.open",
severity: "critical",
suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.",
threshold: 0,
timestamp: new Date(now).toISOString(),
value: Number.isFinite(failureCount) ? failureCount : 1,
window: ALERT_WINDOW_LABEL,
},
{
dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`,
},
);
}
}
private async loadBaselineOverallScore(): Promise<number | null> {
const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY);
const redisParsed = parseBaselineOverallFromRaw(redisBaseline);
if (redisParsed !== null) {
return redisParsed;
}
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
if (!existsSync(filePath)) {
return null;
}
try {
const fileRaw = await readFile(filePath, "utf8");
return parseBaselineOverallFromRaw(fileRaw);
} catch (error) {
this.logger.warn(
{ err: error, filePath },
"[ShadowWorker] Failed to read baseline file",
);
return null;
}
}
}
function parseBaselineOverallFromRaw(raw: string | null): number | null {
if (!raw?.trim()) {
return null;
}
try {
const parsed = JSON.parse(raw) as Partial<EvalBaselineFile>;
const value = parsed.summary?.llmAverageOverall;
return typeof value === "number" ? value : null;
} catch {
return null;
}
}
+151
View File
@@ -0,0 +1,151 @@
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
} from "@agent/types";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
export interface EvalRoomContextMessage {
senderId: string;
nickname: string;
message: string;
timestamp: string;
}
export interface EvalCaseInput {
request: AgentRequest;
conversationHistory?: ConversationMessage[] | undefined;
roomContext?: EvalRoomContextMessage[] | undefined;
session?: OnboardingSession | null | undefined;
blackboard?: Blackboard | null | undefined;
summary?: string | undefined;
}
export interface EvalCaseExpected {
intent?: string | undefined;
mustIncludeAny?: string[] | undefined;
mustNotInclude?: string[] | undefined;
responseMode?: string | undefined;
}
export interface EvalCaseDefinition {
id: string;
lineNumber: number;
name?: string | undefined;
notes?: string | undefined;
input: EvalCaseInput;
expected: EvalCaseExpected;
}
export interface RuleJudgeCheck {
name:
| "validator"
| "intent"
| "responseMode"
| "mustIncludeAny"
| "mustNotInclude";
passed: boolean;
skipped: boolean;
detail?: string | undefined;
}
export interface RuleJudgeResult {
passed: boolean;
score: number;
checks: RuleJudgeCheck[];
validatorStatus: "pass" | "soft-warn" | "reject";
validatorReasons: string[];
}
export interface LlmJudgeScores {
friendliness: number;
accuracy: number;
concision: number;
scope: number;
overall: number;
}
export type LlmJudgeResult =
| {
skipped: true;
reason: string;
raw?: string | undefined;
}
| {
skipped: false;
model: string;
endpoint: string;
provider: string;
scores: LlmJudgeScores;
rationale: string;
raw: string;
};
export interface EvalDerivedSnapshot {
allowShortResponse: boolean;
intent: string;
phase: string;
responseMode: DerivedState["responseMode"];
shouldRespond: boolean;
suggestedMaxTokens: number;
tier: DerivedState["tier"];
}
export interface EvalClassifiedSnapshot {
confidence: number;
intent: string;
source: ClassifiedIntent["source"];
}
export interface EvalCaseResult {
id: string;
lineNumber: number;
name?: string | undefined;
latencyMs: number;
classified: EvalClassifiedSnapshot;
derived: EvalDerivedSnapshot;
request: AgentRequest;
response: AgentResponse;
ruleJudge: RuleJudgeResult;
llmJudge: LlmJudgeResult;
error?: string | undefined;
}
export interface EvalReportSummary {
averageLatencyMs: number;
cases: number;
emptyResponses: number;
llmAverageAccuracy: number | null;
llmAverageConcision: number | null;
llmAverageFriendliness: number | null;
llmAverageOverall: number | null;
llmAverageScope: number | null;
llmJudgedCases: number;
passRate: number;
passedCases: number;
ruleAverageScore: number;
}
export interface EvalSummaryDelta {
averageLatencyMs: number | null;
llmAverageOverall: number | null;
passRate: number | null;
ruleAverageScore: number | null;
}
export interface EvalReport {
baselineDelta?: EvalSummaryDelta | undefined;
generatedAt: string;
inputPath: string;
setName: string;
summary: EvalReportSummary;
cases: EvalCaseResult[];
}
export interface EvalBaselineFile {
reportPath: string;
setName: string;
summary: EvalReportSummary;
updatedAt: string;
}
+3
View File
@@ -1,7 +1,10 @@
export type {
ChatCompletionOptions,
ChatCompletionResult,
ChatCompletionRequest,
ChatCompletionResponse,
ChatMessage,
ChatToolCall,
LlmClientOptions,
LlmEndpointConfig,
} from "./types.js";
+76 -71
View File
@@ -1,49 +1,37 @@
import type {
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionOptions,
ChatMessage,
ChatCompletionResult,
ChatCompletionResponse,
LlmClientOptions,
LlmEndpointConfig,
ProviderName,
} from "./types.js";
import { getRequestLogger } from "@obs/context";
import { withLlmSpan } from "@obs/spans";
import {
getLlmHealthTimeoutMs,
getLlmMaxTokens,
getLlmRequestTimeoutMs,
getLlmTemperature,
} from "../config.js";
import { LlmCircuitBreaker } from "./circuit.js";
import { getProviderAdapter } from "./providers/index.js";
import { selectMaxTokensForTier, selectModelForProvider, selectProviderOrder } from "./routing.js";
const DEFAULT_MODEL = "Qwen3.5-4B";
function buildCliProxyHealthUrl(endpoint: string): string {
try {
const url = new URL(endpoint);
url.pathname = "/";
url.search = "";
url.hash = "";
return url.toString();
} catch {
const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, "");
return `${normalized.replace(/\/+$/u, "")}/`;
}
}
function normalizeEndpointConfig(
endpoint: string | { url: string; provider?: "default" | "cliproxy" },
endpoint: string | { url: string; provider?: ProviderName | "default" | undefined },
): LlmEndpointConfig {
const raw =
typeof endpoint === "string"
? { provider: "default" as const, url: endpoint }
: { provider: endpoint.provider ?? "default", url: endpoint.url };
? { provider: "llama" as const, url: endpoint }
: { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url };
const provider = getProviderAdapter(raw.provider);
return {
completionUrl: buildCompletionUrl(raw.url),
healthUrl:
raw.provider === "cliproxy"
? buildCliProxyHealthUrl(raw.url)
: buildHealthUrl(raw.url),
healthUrl: provider.createHealthUrl(raw.url),
provider: raw.provider,
};
}
@@ -65,49 +53,68 @@ function buildCompletionUrl(endpoint: string): string {
}
}
function buildHealthUrl(endpoint: string): string {
try {
const url = new URL(endpoint);
url.pathname = "/health";
url.search = "";
url.hash = "";
return url.toString();
} catch {
if (endpoint.includes("/v1/chat/completions")) {
return `${endpoint.split("/v1/chat/completions")[0]}/health`;
}
return `${endpoint.replace(/\/+$/u, "")}/health`;
function getRequestSignal(
timeoutMs: number,
externalSignal?: AbortSignal,
): AbortSignal {
const timeoutSignal = AbortSignal.timeout(timeoutMs);
if (!externalSignal) {
return timeoutSignal;
}
if (typeof AbortSignal.any === "function") {
return AbortSignal.any([timeoutSignal, externalSignal]);
}
const controller = new AbortController();
const abort = (): void => controller.abort();
timeoutSignal.addEventListener("abort", abort, { once: true });
externalSignal.addEventListener("abort", abort, { once: true });
return controller.signal;
}
export class LlmClient {
private readonly circuit: LlmCircuitBreaker;
private readonly endpoints: LlmEndpointConfig[];
private readonly apiKey: string | undefined;
private readonly model: string;
private readonly llamaModel: string;
private readonly providerApiKeys: Partial<Record<ProviderName, string>>;
constructor(options: LlmClientOptions) {
this.circuit = new LlmCircuitBreaker(options.garnetClient);
this.endpoints = options.endpoints.map(normalizeEndpointConfig);
this.apiKey = options.apiKey;
this.model = options.model ?? DEFAULT_MODEL;
this.llamaModel = options.model ?? DEFAULT_MODEL;
this.providerApiKeys = options.providerApiKeys ?? {};
}
private getHeaders(includeJson = false): Record<string, string> {
private getHeaders(
provider: ProviderName,
includeJson = false,
): Record<string, string> {
const headers: Record<string, string> = {};
if (includeJson) {
headers["Content-Type"] = "application/json";
}
if (this.apiKey) {
headers["Authorization"] = `Bearer ${this.apiKey}`;
const apiKey = this.providerApiKeys[provider] ?? this.apiKey;
if (apiKey) {
headers["Authorization"] = `Bearer ${apiKey}`;
}
return headers;
}
private async checkHealth(endpoint: LlmEndpointConfig): Promise<boolean> {
const provider = getProviderAdapter(endpoint.provider);
if (!provider.supportsHealthCheck) {
return true;
}
try {
const response = await fetch(endpoint.healthUrl, {
headers: this.getHeaders(),
headers: this.getHeaders(endpoint.provider),
method: "GET",
signal: AbortSignal.timeout(getLlmHealthTimeoutMs()),
});
@@ -122,12 +129,8 @@ export class LlmClient {
async chatCompletion(
messages: ChatMessage[],
options?: {
jsonMode?: boolean | undefined;
maxTokens?: number | undefined;
temperature?: number | undefined;
},
): Promise<{ content: string; endpoint: string } | null> {
options?: ChatCompletionOptions,
): Promise<ChatCompletionResult | null> {
const logger = getRequestLogger();
if (this.endpoints.length === 0) {
@@ -135,18 +138,10 @@ export class LlmClient {
return null;
}
const request: ChatCompletionRequest = {
model: this.model,
messages,
max_tokens: options?.maxTokens ?? getLlmMaxTokens(),
temperature: options?.temperature ?? getLlmTemperature(),
};
const orderedEndpoints = selectProviderOrder(this.endpoints, options);
if (options?.jsonMode) {
request.response_format = { type: "json_object" };
}
for (const endpoint of this.endpoints) {
for (const endpoint of orderedEndpoints) {
const provider = getProviderAdapter(endpoint.provider);
const healthy = await this.circuit.canUse(
endpoint.completionUrl,
() => this.checkHealth(endpoint),
@@ -155,7 +150,7 @@ export class LlmClient {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
},
"[LLM] Endpoint unavailable via circuit breaker",
);
@@ -163,17 +158,30 @@ export class LlmClient {
}
try {
const requestTimeoutMs = Math.max(
1,
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
);
const request = provider.prepareRequest({
jsonMode: options?.jsonMode,
maxTokens: selectMaxTokensForTier(options),
messages,
model: selectModelForProvider(provider.name, options, this.llamaModel),
temperature: options?.temperature ?? getLlmTemperature(),
tools: options?.tools,
});
const llmResult = await withLlmSpan(
{
"gen_ai.request.model": request.model,
"gen_ai.system": endpoint.provider ?? 'default',
"gen_ai.system": provider.name,
},
async (span) => {
const response = await fetch(endpoint.completionUrl, {
body: JSON.stringify(request),
headers: this.getHeaders(true),
headers: this.getHeaders(provider.name, true),
method: "POST",
signal: AbortSignal.timeout(getLlmRequestTimeoutMs()),
signal: getRequestSignal(requestTimeoutMs, options?.signal),
});
if (!response.ok) {
@@ -181,7 +189,7 @@ export class LlmClient {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
status: response.status,
body,
},
@@ -191,7 +199,7 @@ export class LlmClient {
}
const payload = (await response.json()) as ChatCompletionResponse;
const content = payload.choices?.[0]?.message?.content;
const result = provider.toResult(payload, endpoint.completionUrl);
span.setAttributes({
"gen_ai.response.finish_reasons": payload.choices
@@ -201,21 +209,18 @@ export class LlmClient {
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
});
if (typeof content !== "string" || !content.trim()) {
if (!result) {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
},
"[LLM] Empty response from endpoint",
);
return null;
}
return {
content: content.trim(),
endpoint: endpoint.completionUrl,
};
return result;
},
);
@@ -231,7 +236,7 @@ export class LlmClient {
logger.warn(
{
endpoint: endpoint.completionUrl,
provider: endpoint.provider ?? 'default',
provider: provider.name,
error: error instanceof Error ? error.message : String(error),
},
"[LLM] Request error",
+30
View File
@@ -0,0 +1,30 @@
import { buildOpenAiRequest, normalizeProviderResponse } from "./shared.js";
import type { ProviderAdapter } from "./types.js";
export const cliproxyProvider: ProviderAdapter = {
name: "cliproxy",
capabilities: {
jsonMode: true,
promptCache: false,
toolCalls: "openai",
},
createHealthUrl(endpoint: string): string {
try {
const url = new URL(endpoint);
url.pathname = "/";
url.search = "";
url.hash = "";
return url.toString();
} catch {
const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, "");
return `${normalized.replace(/\/+$/u, "")}/`;
}
},
prepareRequest(input) {
return buildOpenAiRequest(input);
},
supportsHealthCheck: true,
toResult(payload, endpoint) {
return normalizeProviderResponse(payload, endpoint, "cliproxy");
},
};
+18
View File
@@ -0,0 +1,18 @@
import type { ProviderName } from "../types.js";
import { cliproxyProvider } from "./cliproxy.js";
import { llamaProvider } from "./llama.js";
import { openrouterProvider } from "./openrouter.js";
import type { ProviderAdapter } from "./types.js";
const PROVIDERS: Record<ProviderName, ProviderAdapter> = {
cliproxy: cliproxyProvider,
llama: llamaProvider,
openrouter: openrouterProvider,
};
export function getProviderAdapter(name: ProviderName): ProviderAdapter {
return PROVIDERS[name];
}
export { cliproxyProvider, llamaProvider, openrouterProvider };
export type { ProviderAdapter, ProviderRequestInput } from "./types.js";
+32
View File
@@ -0,0 +1,32 @@
import { buildHermesRequest, normalizeProviderResponse } from "./shared.js";
import type { ProviderAdapter } from "./types.js";
export const llamaProvider: ProviderAdapter = {
name: "llama",
capabilities: {
jsonMode: false,
promptCache: false,
toolCalls: "hermes",
},
createHealthUrl(endpoint: string): string {
try {
const url = new URL(endpoint);
url.pathname = "/health";
url.search = "";
url.hash = "";
return url.toString();
} catch {
if (endpoint.includes("/v1/chat/completions")) {
return `${endpoint.split("/v1/chat/completions")[0]}/health`;
}
return `${endpoint.replace(/\/+$/u, "")}/health`;
}
},
prepareRequest(input) {
return buildHermesRequest(input);
},
supportsHealthCheck: true,
toResult(payload, endpoint) {
return normalizeProviderResponse(payload, endpoint, "llama");
},
};
+21
View File
@@ -0,0 +1,21 @@
import { buildOpenAiRequest, normalizeProviderResponse } from "./shared.js";
import type { ProviderAdapter } from "./types.js";
export const openrouterProvider: ProviderAdapter = {
name: "openrouter",
capabilities: {
jsonMode: true,
promptCache: false,
toolCalls: "openai",
},
createHealthUrl(endpoint: string): string {
return endpoint;
},
prepareRequest(input) {
return buildOpenAiRequest(input);
},
supportsHealthCheck: false,
toResult(payload, endpoint) {
return normalizeProviderResponse(payload, endpoint, "openrouter");
},
};
+235
View File
@@ -0,0 +1,235 @@
import { serializeToolsForHermes, serializeToolsForOpenAi } from "@tools/serializer";
import type { Tool } from "@tools/tool";
import type {
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResult,
ChatMessage,
ChatToolCall,
} from "../types.js";
import type { ProviderRequestInput } from "./types.js";
function parseJsonSafely(value: string): unknown {
try {
return JSON.parse(value) as unknown;
} catch {
return value;
}
}
function serializeHermesToolCalls(toolCalls: readonly ChatToolCall[]): string {
const payload = toolCalls.map((toolCall) => ({
id: toolCall.id,
name: toolCall.function.name,
arguments: parseJsonSafely(toolCall.function.arguments),
}));
return `<tool_call>${JSON.stringify(payload)}</tool_call>`;
}
function serializeHermesToolResult(message: ChatMessage): string {
return `<tool_result>${JSON.stringify({
name: message.name ?? message.tool_call_id ?? "tool",
result: parseJsonSafely(message.content),
})}</tool_result>`;
}
function injectHermesToolBlock(
messages: ChatMessage[],
tools: readonly Tool<unknown, unknown>[] | undefined,
): ChatMessage[] {
if (!tools || tools.length === 0) {
return [...messages];
}
const toolBlock = serializeToolsForHermes(tools);
const cloned = messages.map((message) => ({ ...message }));
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
if (firstSystemIndex === -1) {
cloned.unshift({
role: "system",
content: toolBlock,
});
return cloned;
}
const current = cloned[firstSystemIndex];
if (!current) {
return cloned;
}
if (current.content.includes("<tools>")) {
return cloned;
}
cloned[firstSystemIndex] = {
...current,
content: `${current.content}\n\n${toolBlock}`,
};
return cloned;
}
function injectJsonModeReminder(messages: ChatMessage[]): ChatMessage[] {
const cloned = messages.map((message) => ({ ...message }));
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
const reminder = "반드시 JSON object 하나만 출력하세요. 설명 문장은 쓰지 마세요.";
if (firstSystemIndex === -1) {
cloned.unshift({
role: "system",
content: reminder,
});
return cloned;
}
const current = cloned[firstSystemIndex];
if (!current || current.content.includes(reminder)) {
return cloned;
}
cloned[firstSystemIndex] = {
...current,
content: `${current.content}\n\n${reminder}`,
};
return cloned;
}
export function normalizeOpenAiMessages(messages: ChatMessage[]): ChatMessage[] {
return messages.map((message) => {
if (message.role === "tool") {
return {
role: "tool",
content: message.content,
tool_call_id: message.tool_call_id,
};
}
return {
role: message.role,
content: message.content,
tool_calls: message.tool_calls,
};
});
}
export function normalizeHermesMessages(messages: ChatMessage[]): ChatMessage[] {
return messages.map((message) => {
if (message.role === "tool") {
return {
role: "user",
content: serializeHermesToolResult(message),
};
}
if (message.tool_calls && message.tool_calls.length > 0) {
const parts = [
message.content.trim(),
serializeHermesToolCalls(message.tool_calls),
].filter(Boolean);
return {
role: message.role,
content: parts.join("\n\n"),
};
}
return {
role: message.role,
content: message.content,
};
});
}
export function buildOpenAiRequest(
input: ProviderRequestInput,
): ChatCompletionRequest {
const request: ChatCompletionRequest = {
model: input.model,
messages: normalizeOpenAiMessages(input.messages),
max_tokens: input.maxTokens,
temperature: input.temperature,
};
if (input.jsonMode) {
request.response_format = { type: "json_object" };
}
if (input.tools && input.tools.length > 0) {
request.tools = serializeToolsForOpenAi(input.tools);
}
return request;
}
export function buildHermesRequest(
input: ProviderRequestInput,
): ChatCompletionRequest {
let messages = injectHermesToolBlock(input.messages, input.tools);
messages = normalizeHermesMessages(messages);
if (input.jsonMode) {
messages = injectJsonModeReminder(messages);
}
return {
model: input.model,
messages,
max_tokens: input.maxTokens,
temperature: input.temperature,
};
}
function normalizeToolCalls(
toolCalls: ChatToolCall[] | null | undefined,
): ChatToolCall[] | undefined {
if (!toolCalls || toolCalls.length === 0) {
return undefined;
}
const normalized = toolCalls.flatMap((toolCall) => {
const name = toolCall.function?.name?.trim();
const args = toolCall.function?.arguments;
if (!name || typeof args !== "string") {
return [];
}
return [
{
id: toolCall.id,
type: "function" as const,
function: {
name,
arguments: args,
},
},
];
});
return normalized.length > 0 ? normalized : undefined;
}
export function normalizeProviderResponse(
payload: ChatCompletionResponse,
endpoint: string,
provider: ChatCompletionResult["provider"],
): ChatCompletionResult | null {
const firstMessage = payload.choices?.[0]?.message;
const content = firstMessage?.content?.trim() ?? "";
const toolCalls = normalizeToolCalls(firstMessage?.tool_calls);
if (!content && (!toolCalls || toolCalls.length === 0)) {
return null;
}
return {
content,
endpoint,
message: {
role: firstMessage?.role ?? "assistant",
content,
tool_calls: toolCalls,
},
provider,
raw: payload,
};
}
+33
View File
@@ -0,0 +1,33 @@
import type { Tool } from "@tools/tool";
import type {
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResult,
ChatMessage,
ProviderName,
} from "../types.js";
export interface ProviderRequestInput {
jsonMode?: boolean | undefined;
maxTokens: number;
messages: ChatMessage[];
model: string;
temperature: number;
tools?: readonly Tool<unknown, unknown>[] | undefined;
}
export interface ProviderAdapter {
capabilities: {
jsonMode: boolean;
promptCache: false;
toolCalls: "openai" | "hermes" | "none";
};
createHealthUrl(endpoint: string): string;
name: ProviderName;
prepareRequest(input: ProviderRequestInput): ChatCompletionRequest;
supportsHealthCheck: boolean;
toResult(
payload: ChatCompletionResponse,
endpoint: string,
): ChatCompletionResult | null;
}
+102
View File
@@ -0,0 +1,102 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { LlmEndpointConfig } from "./types.js";
import {
selectMaxTokensForTier,
selectModelForProvider,
selectProviderOrder,
} from "./routing.js";
describe("llm routing", () => {
afterEach(() => {
delete process.env["OPENROUTER_ENABLED"];
delete process.env["OPENROUTER_TIER1_MODEL"];
delete process.env["OPENROUTER_TIER2_MODEL"];
delete process.env["TIER0_MODEL"];
delete process.env["TIER1_PLANNER_MODEL"];
delete process.env["TIER1_RESPONDER_MODEL"];
delete process.env["TIER2_CRITIC_MODEL"];
vi.restoreAllMocks();
});
it("prefers cliproxy then llama for tier 0 even when openrouter is enabled", () => {
process.env["OPENROUTER_ENABLED"] = "true";
const endpoints: LlmEndpointConfig[] = [
{
completionUrl: "http://llama.test/v1/chat/completions",
healthUrl: "http://llama.test/health",
provider: "llama",
},
{
completionUrl: "http://openrouter.test/v1/chat/completions",
healthUrl: "http://openrouter.test",
provider: "openrouter",
},
{
completionUrl: "http://cliproxy.test/v1/chat/completions",
healthUrl: "http://cliproxy.test",
provider: "cliproxy",
},
];
expect(
selectProviderOrder(endpoints, { tier: 0 }).map((endpoint) => endpoint.provider),
).toEqual(["cliproxy", "llama"]);
});
it("includes openrouter between cliproxy and llama for tier 1+", () => {
process.env["OPENROUTER_ENABLED"] = "true";
const endpoints: LlmEndpointConfig[] = [
{
completionUrl: "http://llama.test/v1/chat/completions",
healthUrl: "http://llama.test/health",
provider: "llama",
},
{
completionUrl: "http://openrouter.test/v1/chat/completions",
healthUrl: "http://openrouter.test",
provider: "openrouter",
},
{
completionUrl: "http://cliproxy.test/v1/chat/completions",
healthUrl: "http://cliproxy.test",
provider: "cliproxy",
},
];
expect(
selectProviderOrder(endpoints, { tier: 1 }).map((endpoint) => endpoint.provider),
).toEqual(["cliproxy", "openrouter", "llama"]);
});
it("selects role-aware models per provider", () => {
process.env["TIER1_PLANNER_MODEL"] = "cliproxy-planner";
process.env["TIER2_CRITIC_MODEL"] = "cliproxy-critic";
process.env["OPENROUTER_TIER1_MODEL"] = "openrouter-tier1";
process.env["OPENROUTER_TIER2_MODEL"] = "openrouter-tier2";
expect(
selectModelForProvider("cliproxy", { role: "planner", tier: 1 }, "local-qwen"),
).toBe("cliproxy-planner");
expect(
selectModelForProvider("cliproxy", { role: "critic", tier: 2 }, "local-qwen"),
).toBe("cliproxy-critic");
expect(selectModelForProvider("openrouter", { tier: 1 }, "local-qwen")).toBe(
"openrouter-tier1",
);
expect(
selectModelForProvider("openrouter", { role: "critic", tier: 2 }, "local-qwen"),
).toBe("openrouter-tier2");
expect(selectModelForProvider("llama", { tier: 2 }, "local-qwen")).toBe("local-qwen");
});
it("chooses tier defaults for max tokens when explicit maxTokens is absent", () => {
process.env["TIER0_MAX_TOKENS"] = "111";
process.env["TIER1_MAX_TOKENS"] = "222";
process.env["TIER2_MAX_TOKENS"] = "333";
expect(selectMaxTokensForTier({ tier: 0 })).toBe(111);
expect(selectMaxTokensForTier({ tier: 1 })).toBe(222);
expect(selectMaxTokensForTier({ tier: 2 })).toBe(333);
expect(selectMaxTokensForTier({ tier: 1, maxTokens: 77 })).toBe(77);
});
});
+99
View File
@@ -0,0 +1,99 @@
import {
getOpenRouterEnabled,
getOpenRouterTier1Model,
getOpenRouterTier2Model,
getTier0MaxTokens,
getTier0Model,
getTier1MaxTokens,
getTier1PlannerModel,
getTier1ResponderModel,
getTier2CriticModel,
getTier2MaxTokens,
getLlmMaxTokens,
} from "../config.js";
import type { ChatCompletionOptions, LlmEndpointConfig, ProviderName } from "./types.js";
export function selectModelForProvider(
provider: ProviderName,
options: ChatCompletionOptions | undefined,
llamaFallbackModel: string,
): string {
if (options?.modelOverride?.trim()) {
return options.modelOverride.trim();
}
if (provider === "llama") {
return llamaFallbackModel;
}
if (provider === "openrouter") {
return options?.role === "critic"
? getOpenRouterTier2Model()
: getOpenRouterTier1Model();
}
if (options?.role === "planner") {
return getTier1PlannerModel();
}
if (options?.role === "critic") {
return getTier2CriticModel();
}
if (options?.tier === 1 || options?.tier === 2) {
return getTier1ResponderModel();
}
return getTier0Model();
}
export function selectMaxTokensForTier(
options: ChatCompletionOptions | undefined,
): number {
if (options?.maxTokens !== undefined) {
return options.maxTokens;
}
if (options?.tier === 0) {
return getTier0MaxTokens();
}
if (options?.tier === 1) {
return getTier1MaxTokens();
}
if (options?.tier === 2) {
return getTier2MaxTokens();
}
return getLlmMaxTokens();
}
export function selectProviderOrder(
endpoints: readonly LlmEndpointConfig[],
options: ChatCompletionOptions | undefined,
): LlmEndpointConfig[] {
const preferredProviders: ProviderName[] = ["cliproxy"];
if (getOpenRouterEnabled() && options?.tier !== 0) {
preferredProviders.push("openrouter");
}
preferredProviders.push("llama");
const weighted = endpoints.map((endpoint, index) => ({
endpoint,
index,
priority: preferredProviders.indexOf(endpoint.provider),
}));
return weighted
.filter((item) => item.priority !== -1)
.sort((left, right) => {
if (left.priority !== right.priority) {
return left.priority - right.priority;
}
return left.index - right.index;
})
.map((item) => item.endpoint);
}
+44 -3
View File
@@ -1,8 +1,23 @@
import type { GarnetClient } from '@common/garnet';
import type { Tool } from '@tools/tool';
export type ProviderName = "cliproxy" | "llama" | "openrouter";
export interface ChatToolCall {
id?: string | undefined;
type: "function";
function: {
name: string;
arguments: string;
};
}
export interface ChatMessage {
role: "system" | "user" | "assistant";
role: "system" | "user" | "assistant" | "tool";
content: string;
name?: string | undefined;
tool_call_id?: string | undefined;
tool_calls?: ChatToolCall[] | undefined;
}
export interface ChatCompletionRequest {
@@ -11,6 +26,7 @@ export interface ChatCompletionRequest {
max_tokens: number;
temperature: number;
response_format?: { type: string } | undefined;
tools?: unknown[] | undefined;
}
export interface ChatCompletionResponse {
@@ -18,7 +34,9 @@ export interface ChatCompletionResponse {
choices?: Array<{
finish_reason?: string | null;
message?: {
role?: "assistant" | "tool" | "user" | "system" | null;
content?: string | null;
tool_calls?: ChatToolCall[] | null;
};
}>;
usage?: {
@@ -28,15 +46,38 @@ export interface ChatCompletionResponse {
};
}
export interface ChatCompletionResult {
content: string;
endpoint: string;
message: ChatMessage;
provider: ProviderName;
raw: ChatCompletionResponse;
}
export interface ChatCompletionOptions {
deadlineMs?: number | undefined;
jsonMode?: boolean | undefined;
maxTokens?: number | undefined;
modelOverride?: string | undefined;
role?: "planner" | "responder" | "critic" | undefined;
signal?: AbortSignal | undefined;
temperature?: number | undefined;
tier?: 0 | 1 | 2 | undefined;
tools?: readonly Tool<unknown, unknown>[] | undefined;
}
export interface LlmEndpointConfig {
completionUrl: string;
healthUrl: string;
provider?: "default" | "cliproxy" | undefined;
provider: ProviderName;
}
export interface LlmClientOptions {
endpoints: Array<string | { url: string; provider?: "default" | "cliproxy" }>;
endpoints: Array<
string | { url: string; provider?: ProviderName | "default" | undefined }
>;
garnetClient?: GarnetClient | undefined;
apiKey?: string | undefined;
model?: string | undefined;
providerApiKeys?: Partial<Record<ProviderName, string>> | undefined;
}
+33 -11
View File
@@ -1,5 +1,5 @@
import { loadEnvironmentFromNodeEnv } from "./common/env-loader.js";
import { getServiceName } from "./config.js";
import { getOpenRouterApiUrl, getOpenRouterEnabled, getServiceName } from "./config.js";
import type { GarnetClient } from "./common/garnet.js";
import type { NatsClient } from "./common/nats.js";
@@ -21,6 +21,7 @@ const { NatsClient: RuntimeNatsClient } = await import("./common/nats.js");
const { Agent } = await import("./agent/index.js");
const { ConversationStore } = await import("./conversation/index.js");
const { OnboardingSkill } = await import("./skills/onboarding/index.js");
const { ShadowEvalBus } = await import("./eval/shadow-bus.js");
const { LlmClient } = await import("./llm/index.js");
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
@@ -53,22 +54,22 @@ function parseEndpointList(
function toProviderEndpoints(
urls: string[],
provider?: "default" | "cliproxy",
): Array<string | { url: string; provider?: "default" | "cliproxy" }> {
if (!provider || provider === "default") {
provider?: "llama" | "cliproxy" | "openrouter",
): Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }> {
if (!provider) {
return urls;
}
return urls.map((url) => ({ url, provider }));
}
function formatEndpointList(
endpoints: Array<string | { url: string; provider?: "default" | "cliproxy" }>,
endpoints: Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }>,
): string {
return endpoints
.map((endpoint) =>
typeof endpoint === "string"
? endpoint
: `${endpoint.provider ?? "default"}:${endpoint.url}`,
: `${endpoint.provider}:${endpoint.url}`,
)
.join(", ");
}
@@ -135,30 +136,50 @@ async function bootstrap(): Promise<void> {
getRawSecret("LLAMA_SERVER_API_URL"),
process.env["LLAMA_SERVER_API_URL"],
);
const openRouterEndpoints = getOpenRouterEnabled()
? parseEndpointList(
getRawSecret("OPENROUTER_API_URL"),
process.env["OPENROUTER_API_URL"],
getOpenRouterApiUrl(),
)
: [];
const llmEndpoints = [
...toProviderEndpoints(cliProxyEndpoints, "cliproxy"),
...toProviderEndpoints(generalLlmEndpoints),
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
...toProviderEndpoints(generalLlmEndpoints, "llama"),
];
const llmApiKey = firstNonEmpty(
const cliproxyApiKey = firstNonEmpty(
getRawSecret("CLI_PROXY_API_KEY"),
process.env["CLI_PROXY_API_KEY"],
);
const openRouterApiKey = firstNonEmpty(
getRawSecret("OPENROUTER_API_KEY"),
process.env["OPENROUTER_API_KEY"],
);
const llmApiKey = firstNonEmpty(
getRawSecret("AGENT_LLM_API_KEY"),
process.env["AGENT_LLM_API_KEY"],
getRawSecret("LLAMA_SERVER_API_KEY"),
process.env["LLAMA_SERVER_API_KEY"],
);
const llmModel = firstNonEmpty(
getRawSecret("CLI_PROXY_LLM_MODEL"),
process.env["CLI_PROXY_LLM_MODEL"],
getRawSecret("AGENT_LLM_MODEL"),
process.env["AGENT_LLM_MODEL"],
);
const providerApiKeys = Object.fromEntries(
[
["cliproxy", cliproxyApiKey],
["llama", llmApiKey],
["openrouter", openRouterApiKey],
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
) as Partial<Record<"cliproxy" | "llama" | "openrouter", string>>;
const llmClient = new LlmClient({
endpoints: llmEndpoints,
garnetClient,
apiKey: llmApiKey,
model: llmModel,
providerApiKeys,
});
if (llmEndpoints.length > 0) {
@@ -197,7 +218,8 @@ async function bootstrap(): Promise<void> {
const agent = new Agent(natsClient, conversationStore);
// Register skills
const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService);
const shadowEvalBus = new ShadowEvalBus(garnetClient);
const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService, shadowEvalBus);
agent.registerSkill(onboardingSkill);
installShutdownHooks(async () => {
+40
View File
@@ -0,0 +1,40 @@
import { describe, expect, it, vi } from "vitest";
import type { AlertPayload } from "@common/nats-subjects";
import { AGENT_ALERT_SUBJECT } from "@common/nats-subjects";
import { AlertService } from "./alert.js";
const payload: AlertPayload = {
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: "warn",
threshold: 3000,
timestamp: "2026-04-17T00:00:00.000Z",
value: 4100,
window: "5m",
};
describe("AlertService", () => {
it("publishes when dedupe key is claimed", async () => {
const setStringIfAbsent = vi.fn().mockResolvedValue(true);
const publish = vi.fn().mockResolvedValue(undefined);
const service = new AlertService({ setStringIfAbsent }, { publish });
const result = await service.sendAlert(payload);
expect(result).toBe(true);
expect(setStringIfAbsent).toHaveBeenCalledTimes(1);
expect(publish).toHaveBeenCalledWith(AGENT_ALERT_SUBJECT, payload);
});
it("suppresses duplicate alerts", async () => {
const setStringIfAbsent = vi.fn().mockResolvedValue(false);
const publish = vi.fn().mockResolvedValue(undefined);
const service = new AlertService({ setStringIfAbsent }, { publish });
const result = await service.sendAlert(payload);
expect(result).toBe(false);
expect(publish).not.toHaveBeenCalled();
});
});
+66
View File
@@ -0,0 +1,66 @@
import type { GarnetClient } from "@common/garnet";
import { AGENT_ALERT_SUBJECT, type AlertPayload } from "@common/nats-subjects";
import type { NatsClient } from "@common/nats";
import { getAlertRateLimitSec } from "../config.js";
import { getLogger } from "./logger.js";
export interface AlertSender {
sendAlert(
payload: AlertPayload,
options?: {
dedupeKey?: string | undefined;
},
): Promise<boolean>;
}
function getAlertDedupeKey(kind: AlertPayload["kind"]): string {
return `lolympus-agent:alert:dedupe:${kind}`;
}
export class AlertService implements AlertSender {
private readonly logger = getLogger();
constructor(
private readonly garnetClient: Pick<GarnetClient, "setStringIfAbsent">,
private readonly natsClient: Pick<NatsClient, "publish">,
) {}
async sendAlert(
payload: AlertPayload,
options?: {
dedupeKey?: string | undefined;
},
): Promise<boolean> {
const dedupeKey = options?.dedupeKey ?? getAlertDedupeKey(payload.kind);
const claimed = await this.garnetClient.setStringIfAbsent(
dedupeKey,
payload.timestamp,
getAlertRateLimitSec(),
);
if (!claimed) {
this.logger.debug(
{
dedupeKey,
kind: payload.kind,
},
"[Alert] Suppressed by rate limit",
);
return false;
}
await this.natsClient.publish(AGENT_ALERT_SUBJECT, payload);
this.logger.warn(
{
kind: payload.kind,
metric: payload.metric,
severity: payload.severity,
threshold: payload.threshold,
value: payload.value,
window: payload.window,
},
"[Alert] Published",
);
return true;
}
}
+9 -2
View File
@@ -17,11 +17,18 @@ export function initTracing(): void {
return;
}
const exporterType = getOtelExporter();
if (exporterType === 'none') {
const logger = getLogger();
logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init');
started = true;
return;
}
process.env['OTEL_SERVICE_NAME'] ??= getServiceName();
const logger = getLogger();
const exporter =
getOtelExporter() === 'stdout' ? new ConsoleSpanExporter() : new ConsoleSpanExporter();
const exporter = new ConsoleSpanExporter();
sdk = new NodeSDK({
instrumentations: [getNodeAutoInstrumentations()],
+45
View File
@@ -0,0 +1,45 @@
export const VALID_TIERS = [
"GM",
"I",
"B",
"S",
"G",
"P",
"E",
"D",
"M",
"C",
"U",
] as const;
export const VALID_TIER_SET = new Set<string>(VALID_TIERS);
export const VALID_GENDERS = ["남", "여"] as const;
export const VALID_GENDER_SET = new Set<string>(VALID_GENDERS);
export const MIN_AGE = 27;
export const MAX_AGE: number | null = null;
export const NICKNAME_FORMAT_EXAMPLE = "홍길동/95/남/G" as const;
export const TIER_CODES_TEXT = VALID_TIERS.join(", ");
export const NICKNAME_FORMAT_TEXT =
`닉네임은 이름/출생 연도 두 자리/성별/티어 형식이에요. 예를 들면 ${NICKNAME_FORMAT_EXAMPLE}처럼 쓰시면 돼요.`;
export const HEART_RULE_TEXT =
'채팅방 오른쪽 상단 메뉴 버튼에서 "커버 보기"로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.';
export const HEART_LOCATION_TEXT =
'채팅방 오른쪽 상단 메뉴 버튼을 누르면 "커버 보기"가 보여요. 거기서 프로필과 하트를 확인하실 수 있어요.';
export const DISCORD_GUIDE_TEXT =
"닉네임 변경과 하트 인증을 모두 완료하신 뒤에 디스코드 서버에서 프로필을 설정하시면 권한이 자동으로 부여돼요.";
export const DISCORD_INVITE_URL =
process.env["DISCORD_INVITE_URL"]?.trim() || "";
export const DEADLINE_NEAR_THRESHOLD_MIN = Number(
process.env["DEADLINE_NEAR_THRESHOLD_MIN"] ?? 30,
);
export const ESCALATION_BUDGET_PER_TURN = Number(
process.env["ESCALATION_BUDGET_PER_TURN"] ?? 2,
);
+21 -14
View File
@@ -1,12 +1,19 @@
import type { NicknameIssue, OnboardingSession } from "./types.js";
import {
HEART_LOCATION_TEXT,
HEART_RULE_TEXT,
MIN_AGE,
NICKNAME_FORMAT_EXAMPLE,
NICKNAME_FORMAT_TEXT,
TIER_CODES_TEXT,
VALID_GENDER_SET,
VALID_TIER_SET,
} from "./constants.js";
function pickRandom(items: [string, ...string[]]): string {
return items[Math.floor(Math.random() * items.length)] ?? items[0];
}
const VALID_GENDERS = new Set(["남", "여"]);
const VALID_TIERS = new Set(["GM", "I", "B", "S", "G", "P", "E", "D", "M", "C", "U"]);
export function analyzeNicknameIssues(nickname: string): NicknameIssue[] {
const parts = nickname.trim().split("/");
const issues: NicknameIssue[] = [];
@@ -21,9 +28,9 @@ export function analyzeNicknameIssues(nickname: string): NicknameIssue[] {
});
}
if (third) {
const hasGender = VALID_GENDERS.has(third.charAt(0));
const hasGender = VALID_GENDER_SET.has(third.charAt(0));
const rest = third.slice(1);
if (hasGender && rest.length > 0 && !VALID_TIERS.has(rest)) {
if (hasGender && rest.length > 0 && !VALID_TIER_SET.has(rest)) {
issues.push({
kind: "unknown-tier",
message: `성별 뒤의 '${rest}'를 제거하고, 티어 코드를 '/'로 구분해서 추가해주세요 (예: ${third.charAt(0)}/D)`,
@@ -74,17 +81,17 @@ export function analyzeNicknameIssues(nickname: string): NicknameIssue[] {
});
}
if (!gender || !VALID_GENDERS.has(gender)) {
if (!gender || !VALID_GENDER_SET.has(gender)) {
issues.push({
kind: "missing-gender",
message: "성별을 '남' 또는 '여'로 입력해주세요",
});
}
if (!tier || !VALID_TIERS.has(tier)) {
if (!tier || !VALID_TIER_SET.has(tier)) {
issues.push({
kind: "unknown-tier",
message: "티어 코드(GM,I,B,S,G,P,E,D,M,C,U 중 하나)를 입력해주세요",
message: `티어 코드(${TIER_CODES_TEXT} 중 하나)를 입력해주세요`,
});
}
@@ -146,29 +153,29 @@ export function getNicknameInvalidMessage(
export function getNicknameFormatMessage(): string {
return pickRandom([
"닉네임 형식은 이름/출생 연도 두 자리/성별/티어예요. 예를 들면 홍길동/95/여/D처럼 쓰시면 돼요.",
"닉네임은 이름/출생 연도 두 자리/성별/티어 형식으로 입력해 주시면 돼요! 예: 홍길동/95/남/G",
NICKNAME_FORMAT_TEXT,
`닉네임은 이름/출생 연도 두 자리/성별/티어 형식으로 입력해 주시면 돼요! 예: ${NICKNAME_FORMAT_EXAMPLE}`,
]);
}
export function getHeartGuideMessage(): string {
return pickRandom([
'채팅방 오른쪽 상단 메뉴 버튼에서 "커버 보기"로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.',
HEART_RULE_TEXT,
"채팅방 프로필에서 하트를 눌러 빨간색으로 만든 다음, 그 화면이 보이게 원본 스크린샷을 보내주시면 인증돼요.",
]);
}
export function getHeartLocationMessage(): string {
return pickRandom([
'채팅방 오른쪽 상단 메뉴 버튼을 누르면 "커버 보기"가 보여요. 거기서 프로필과 하트를 확인하실 수 있어요.',
HEART_LOCATION_TEXT,
'오른쪽 상단 메뉴 버튼으로 들어가 "커버 보기"를 열면 하트 위치를 확인하실 수 있어요.',
]);
}
export function getAgeRequirementMessage(): string {
return pickRandom([
"롤림푸스는 27세 이상만 가입이 가능해요.",
"가입 가능 연령은 27세 이상이에요.",
`롤림푸스는 ${MIN_AGE}세 이상만 가입이 가능해요.`,
`가입 가능 연령은 ${MIN_AGE}세 이상이에요.`,
]);
}
+7 -1
View File
@@ -8,6 +8,7 @@ import {
policyLayer,
stateLayer,
} from "./layers.js";
import { buildToolPolicy } from "./tool-policy.js";
export class PromptBuilder {
private constructor(
@@ -35,7 +36,12 @@ export class PromptBuilder {
const systemPrompt = [
personaLayer(),
policyLayer(this.derived.intent),
policyLayer(this.derived.intent, {
toolPolicy:
this.derived.responseMode === "llm-with-tools"
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
: undefined,
}),
stateLayer(this.session, this.derived),
examplesLayer(this.derived.intent),
contextLayer(promptContext),
+10 -4
View File
@@ -7,6 +7,12 @@ import type {
OnboardingSession,
PromptContext,
} from "../types.js";
import {
HEART_RULE_TEXT,
MIN_AGE,
NICKNAME_FORMAT_EXAMPLE,
TIER_CODES_TEXT,
} from "../constants.js";
import { EXAMPLES_BY_INTENT } from "./examples.js";
function formatKstTimestamp(value: string | undefined): string {
@@ -94,11 +100,11 @@ export function personaLayer(): string {
처음 온 멤버가 편안하게 등록을 마칠 수 있도록 자연스럽고 정확하게 안내해줘.
[커뮤니티 규칙]
- 가입 가능 연령은 27세 이상이야.
- 가입 가능 연령은 ${MIN_AGE}세 이상이야.
- 새내기 등록은 닉네임 변경과 방 하트 인증 두 가지를 모두 완료하면 돼.
- 닉네임 형식은 이름/출생 연도 두 자리/성별/티어야. 예: 홍길동/95/남/G
- 성별은 남 또는 여, 티어 코드는 GM, I, B, S, G, P, E, D, M, C, U 중 하나야.
- 방 하트 인증은 채팅방 프로필에서 하트를 빨간색으로 바꾼 뒤 편집 없는 원본 스크린샷을 보내는 절차야.
- 닉네임 형식은 이름/출생 연도 두 자리/성별/티어야. 예: ${NICKNAME_FORMAT_EXAMPLE}
- 성별은 남 또는 여, 티어 코드는 ${TIER_CODES_TEXT} 중 하나야.
- 방 하트 인증은 ${HEART_RULE_TEXT}
- 디스코드 안내는 닉네임 변경과 하트 인증이 모두 완료된 뒤에만 해줘.`;
}
@@ -0,0 +1,28 @@
import type { Intent, OnboardingPhase } from "../state/onboarding-state.js";
export function buildToolPolicy(
intent: Intent["kind"],
phase: OnboardingPhase["kind"],
): string {
if (intent === "nickname-question" && phase === "nickname-rejected") {
return "- 닉네임 후보나 현재 닉네임을 설명할 때는 validate-nickname 도구를 먼저 사용한 뒤, 틀린 부분만 짚어.";
}
if (intent === "status-question") {
return "- 진행 상태 질문은 check-onboarding-progress 도구를 우선 사용하고, 상태만 짧게 답해.";
}
if (intent === "age-question" || intent === "discord-question") {
return "- 규칙 확인이 필요하면 lookup-community-rule 도구를 사용해 정확한 규칙만 답해.";
}
if (intent === "help-where") {
return "- 위치 안내가 모호할 때만 get-room-context 도구를 고려하고, 기본적으로는 절차 위치만 답해.";
}
if (phase === "deadline-near") {
return "- 마감이 가까운 상태에서는 남은 절차만 우선순위로 짧게 답해.";
}
return "- 도구가 꼭 필요하지 않다면 호출하지 말고, 지금 질문에 대한 답만 간결하게 해.";
}
+83
View File
@@ -0,0 +1,83 @@
import type { ChatMessage } from "@llm/types";
import type { LlmClient } from "@llm/llm-client";
import {
getCriticMaxTokens,
getLlmTemperature,
getTier2CriticModel,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
export interface CritiqueResult {
changed: boolean;
text: string;
}
function serializeMessages(messages: ChatMessage[]): string {
return messages
.map((message) => `[${message.role}] ${message.content}`)
.join("\n\n");
}
function buildCriticMessages(
draft: string,
derived: DerivedState,
messages: ChatMessage[],
): ChatMessage[] {
return [
{
role: "system",
content: [
"당신은 온보딩 응답을 검토하는 critic입니다.",
"초안이 충분히 정확하고 범위를 벗어나지 않으며 간결하면 반드시 [OK]만 출력하세요.",
"고쳐야 한다면 유저에게 바로 보낼 최종 한국어 답변만 출력하고, 설명이나 해설은 쓰지 마세요.",
"질문에서 묻지 않은 추가 절차를 길게 덧붙이지 말고, 꼭 필요한 내용만 남기세요.",
].join("\n"),
},
{
role: "user",
content: [
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}`,
"[CONVERSATION]",
serializeMessages(messages),
"[DRAFT]",
draft,
].join("\n\n"),
},
];
}
export class OnboardingCritic {
constructor(private readonly llmClient: LlmClient) {}
async critique(
draft: string,
derived: DerivedState,
messages: ChatMessage[],
options?: { deadlineMs?: number | undefined },
): Promise<CritiqueResult> {
const response = await this.llmClient.chatCompletion(
buildCriticMessages(draft, derived, messages),
{
deadlineMs: options?.deadlineMs,
maxTokens: getCriticMaxTokens(),
modelOverride: getTier2CriticModel(),
role: "critic",
temperature: Math.max(0, getLlmTemperature() - 0.15),
tier: 2,
},
);
const text = response?.content.trim() ?? "";
if (!text || text.startsWith("[OK]")) {
return {
changed: false,
text: draft,
};
}
return {
changed: true,
text,
};
}
}
@@ -0,0 +1,182 @@
import { describe, expect, it } from "vitest";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { OnboardingOrchestrator } from "./orchestrator.js";
import type { DerivedState } from "../state/onboarding-state.js";
function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
return {
phase: { kind: "photo-pending" },
intent: { kind: "help-how" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 256,
allowShortResponse: false,
tier: 0,
...overrides,
};
}
describe("OnboardingOrchestrator", () => {
it("uses direct response path for tier 0", async () => {
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
return Promise.resolve({
content: "안내 도와드릴게요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "안내 도와드릴게요." } }] },
message: { role: "assistant" as const, content: "안내 도와드릴게요." },
});
},
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ tier: 0 }),
[{ role: "user", content: "도와줘" }],
{ toolContext: {} },
);
expect(result).toBe("안내 도와드릴게요.");
expect(calls).toHaveLength(1);
expect(calls[0]?.options?.role).toBe("responder");
});
it("runs planner then responder for tier 1", async () => {
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
if (calls.length === 1) {
return Promise.resolve({
content: JSON.stringify({
approach: "redirect-to-step",
targetPhase: "photo-pending",
mustInclude: ["하트 인증만 안내"],
mustAvoid: ["디스코드 절차"],
suggestedTools: [],
tone: "friendly",
targetLengthChars: 120,
rationale: "질문 범위만 답하기",
}),
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "{}" } }] },
message: { role: "assistant" as const, content: "{}" },
});
}
return Promise.resolve({
content: "하트 인증만 남았어요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
});
},
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ tier: 1 }),
[
{ role: "system", content: "기본 시스템 프롬프트" },
{ role: "user", content: "이제 뭐 남았지" },
],
{ toolContext: {} },
);
expect(result).toBe("하트 인증만 남았어요.");
expect(calls).toHaveLength(2);
expect(calls[0]?.options?.role).toBe("planner");
expect(calls[1]?.messages[0]?.content).toContain("[PLAN]");
});
it("runs critic after responder for tier 2 and applies rewrite", async () => {
const calls: Array<{
messages: ChatMessage[];
options: ChatCompletionOptions | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: ChatMessage[],
options?: ChatCompletionOptions,
) => {
calls.push({ messages, options });
if (calls.length === 1) {
return Promise.resolve({
content: JSON.stringify({
approach: "answer-direct",
targetPhase: "photo-pending",
mustInclude: [],
mustAvoid: ["추가 절차 나열"],
suggestedTools: [],
tone: "concise",
targetLengthChars: 60,
rationale: "짧게 답하기",
}),
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "{}" } }] },
message: { role: "assistant" as const, content: "{}" },
});
}
if (calls.length === 2) {
return Promise.resolve({
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: {
choices: [
{ message: { content: "하트 인증만 남았어요. 커버 보기로 들어가서..." } },
],
},
message: {
role: "assistant" as const,
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
},
});
}
return Promise.resolve({
content: "하트 인증만 남았어요.",
endpoint: "mock://llm",
provider: "cliproxy" as const,
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
});
},
};
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
const result = await orchestrator.run(
createDerivedState({ tier: 2 }),
[
{ role: "system", content: "기본 시스템 프롬프트" },
{ role: "user", content: "이제 뭐 남았지" },
],
{ toolContext: {} },
);
expect(result).toBe("하트 인증만 남았어요.");
expect(calls).toHaveLength(3);
expect(calls[2]?.options?.role).toBe("critic");
});
});
@@ -0,0 +1,382 @@
import { getContext } from "@obs/context";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { Tool, ToolContext } from "@tools/tool";
import {
getLlmTemperature,
getPlannerMaxTokens,
getTier0EstimatedCallMs,
getTier0MaxTokens,
getTier0Model,
getTier1EstimatedCallMs,
getTier1MaxTokens,
getTier1PlannerModel,
getTier1ResponderModel,
getTier2EstimatedCallMs,
getTier2MaxTokens,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
import { OnboardingCritic } from "./critic.js";
import type { PlannerOutput } from "./planner-schema.js";
import { PlannerOutputSchema } from "./planner-schema.js";
import { runWithTools } from "./tool-loop.js";
export interface OrchestratorRunOptions {
maxTokens?: number | undefined;
temperature?: number | undefined;
toolContext: ToolContext;
tools?: readonly Tool<unknown, unknown>[] | undefined;
}
function cloneMessages(messages: ChatMessage[]): ChatMessage[] {
return messages.map((message) => ({
...message,
tool_calls: message.tool_calls ? [...message.tool_calls] : undefined,
}));
}
function injectSystemAddendum(
messages: ChatMessage[],
addendum: string,
): ChatMessage[] {
const cloned = cloneMessages(messages);
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
if (firstSystemIndex === -1) {
cloned.unshift({
role: "system",
content: addendum,
});
return cloned;
}
const current = cloned[firstSystemIndex];
if (!current) {
return cloned;
}
cloned[firstSystemIndex] = {
...current,
content: `${current.content}\n\n${addendum}`,
};
return cloned;
}
function summarizeMessages(messages: ChatMessage[]): string {
return messages
.map((message) => `[${message.role}] ${message.content}`)
.join("\n\n");
}
function tryParsePlannerOutput(raw: string): PlannerOutput | null {
try {
const parsed = JSON.parse(raw) as unknown;
return PlannerOutputSchema.parse(parsed);
} catch {
const start = raw.indexOf("{");
const end = raw.lastIndexOf("}");
if (start === -1 || end === -1 || end <= start) {
return null;
}
try {
const parsed = JSON.parse(raw.slice(start, end + 1)) as unknown;
return PlannerOutputSchema.parse(parsed);
} catch {
return null;
}
}
}
function getDefaultPlan(derived: DerivedState): PlannerOutput {
return {
approach:
derived.responseMode === "llm-with-tools"
? "validate-then-explain"
: derived.intent.kind === "help-how" || derived.intent.kind === "help-where"
? "redirect-to-step"
: "answer-direct",
targetPhase: derived.phase.kind,
mustInclude: [],
mustAvoid: [],
suggestedTools:
derived.responseMode === "llm-with-tools" ? ["validate-nickname"] : [],
tone: derived.allowShortResponse ? "concise" : "friendly",
targetLengthChars: derived.allowShortResponse ? 80 : 150,
rationale: "fallback-plan",
};
}
function getRemainingDeadlineMs(): number | undefined {
const deadlineAt = getContext()?.deadlineAt;
if (!deadlineAt) {
return undefined;
}
const deadlineMs = Date.parse(deadlineAt) - Date.now();
return Number.isFinite(deadlineMs) ? deadlineMs : undefined;
}
function canAffordCall(estimatedCallMs: number): boolean {
const remainingMs = getRemainingDeadlineMs();
return remainingMs === undefined || remainingMs > estimatedCallMs;
}
function buildPlannerMessages(
derived: DerivedState,
messages: ChatMessage[],
): ChatMessage[] {
return [
{
role: "system",
content: [
"당신은 온보딩 응답을 설계하는 planner입니다.",
"반드시 JSON만 출력하세요.",
"질문에 직접 답할 수 있으면 answer-direct를 우선하고, 유저가 묻지 않은 추가 절차는 mustAvoid에 넣으세요.",
"tool은 꼭 필요한 경우에만 suggestedTools에 넣고 최대 3개까지만 고르세요.",
].join("\n"),
},
{
role: "user",
content: [
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}, tier=${derived.tier}`,
"[CONVERSATION]",
summarizeMessages(messages),
"[SCHEMA]",
JSON.stringify({
approach:
"answer-direct | ask-clarification | redirect-to-step | validate-then-explain | lookup-rule-then-answer | soft-decline",
targetPhase:
"fresh | nickname-pending | photo-pending | both-pending | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
mustInclude: ["short strings"],
mustAvoid: ["short strings"],
suggestedTools: [
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context",
],
tone: "friendly | concise | encouraging | neutral | apologetic",
targetLengthChars: 150,
rationale: "short reason",
}),
].join("\n\n"),
},
];
}
function buildResponderAddendum(plan: PlannerOutput): string {
const lines = [
"[PLAN]",
`approach=${plan.approach}`,
`tone=${plan.tone}`,
`targetLengthChars=${plan.targetLengthChars}`,
];
if (plan.targetPhase) {
lines.push(`targetPhase=${plan.targetPhase}`);
}
lines.push(
`mustInclude=${plan.mustInclude.length > 0 ? plan.mustInclude.join(" | ") : "-"}`,
`mustAvoid=${plan.mustAvoid.length > 0 ? plan.mustAvoid.join(" | ") : "-"}`,
"위 계획을 참고하되, 실제 유저에게 보낼 최종 한국어 답변만 출력하세요.",
);
return lines.join("\n");
}
function pickToolsForPlan(
plan: PlannerOutput,
tools: readonly Tool<unknown, unknown>[],
): readonly Tool<unknown, unknown>[] {
if (plan.approach === "answer-direct" || plan.approach === "soft-decline") {
return [];
}
if (plan.suggestedTools.length === 0) {
return tools;
}
const picked = tools.filter((tool) => plan.suggestedTools.includes(tool.name as never));
return picked.length > 0 ? picked : tools;
}
export class OnboardingOrchestrator {
private readonly critic: OnboardingCritic;
constructor(private readonly llmClient: LlmClient) {
this.critic = new OnboardingCritic(llmClient);
}
private async runDirect(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
responderTools: readonly Tool<unknown, unknown>[],
maxTokens: number,
modelOverride: string,
): Promise<string | null> {
const deadlineMs = getRemainingDeadlineMs();
if (responderTools.length > 0) {
return runWithTools({
deadlineMs,
llmClient: this.llmClient,
maxTokens,
messages,
modelOverride,
role: "responder",
tier: derived.tier,
temperature: options.temperature ?? getLlmTemperature(),
toolContext: options.toolContext,
tools: responderTools,
});
}
const response = await this.llmClient.chatCompletion(messages, {
deadlineMs,
maxTokens,
modelOverride,
role: "responder",
temperature: options.temperature ?? getLlmTemperature(),
tier: derived.tier,
});
return response?.content ?? null;
}
private async runPlanner(
derived: DerivedState,
messages: ChatMessage[],
): Promise<PlannerOutput> {
const plannerMessages = buildPlannerMessages(derived, messages);
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
deadlineMs: getRemainingDeadlineMs(),
jsonMode: true,
maxTokens: getPlannerMaxTokens(),
modelOverride: getTier1PlannerModel(),
role: "planner",
temperature: 0,
tier: 1,
});
const firstPlan = firstAttempt?.content
? tryParsePlannerOutput(firstAttempt.content)
: null;
if (firstPlan) {
return firstPlan;
}
const retryMessages = [
...plannerMessages,
{
role: "user" as const,
content:
"방금 응답은 스키마를 벗어났어요. 설명 없이 JSON object 하나만 다시 출력하세요.",
},
];
const retry = await this.llmClient.chatCompletion(retryMessages, {
deadlineMs: getRemainingDeadlineMs(),
jsonMode: true,
maxTokens: getPlannerMaxTokens(),
modelOverride: getTier1PlannerModel(),
role: "planner",
temperature: 0,
tier: 1,
});
const retryPlan = retry?.content ? tryParsePlannerOutput(retry.content) : null;
return retryPlan ?? getDefaultPlan(derived);
}
private async runTier0(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
const maxTokens = options.maxTokens ?? getTier0MaxTokens();
const tools =
derived.responseMode === "llm-with-tools" ? (options.tools ?? []) : [];
return this.runDirect(
derived,
messages,
options,
tools,
maxTokens,
getTier0Model(),
);
}
private async runTier1(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
const plan = await this.runPlanner(derived, messages);
const responderMessages = injectSystemAddendum(
messages,
buildResponderAddendum(plan),
);
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
return this.runDirect(
derived,
responderMessages,
options,
responderTools,
options.maxTokens ?? getTier1MaxTokens(),
getTier1ResponderModel(),
);
}
private async runTier2(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
const plan = await this.runPlanner(derived, messages);
const responderMessages = injectSystemAddendum(
messages,
buildResponderAddendum(plan),
);
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
const draft = await this.runDirect(
derived,
responderMessages,
options,
responderTools,
options.maxTokens ?? getTier2MaxTokens(),
getTier1ResponderModel(),
);
if (!draft) {
return null;
}
const critique = await this.critic.critique(draft, derived, responderMessages, {
deadlineMs: getRemainingDeadlineMs(),
});
return critique.text;
}
async run(
derived: DerivedState,
messages: ChatMessage[],
options: OrchestratorRunOptions,
): Promise<string | null> {
if (derived.tier >= 2 && canAffordCall(getTier2EstimatedCallMs())) {
return this.runTier2(derived, messages, options);
}
if (derived.tier >= 1 && canAffordCall(getTier1EstimatedCallMs())) {
return this.runTier1(derived, messages, options);
}
if (!canAffordCall(getTier0EstimatedCallMs())) {
return null;
}
return this.runTier0(derived, messages, options);
}
}
@@ -0,0 +1,50 @@
import { z } from "zod";
export const PlannerApproachSchema = z.enum([
"answer-direct",
"ask-clarification",
"redirect-to-step",
"validate-then-explain",
"lookup-rule-then-answer",
"soft-decline",
]);
export const PlannerTargetPhaseSchema = z.enum([
"fresh",
"nickname-pending",
"photo-pending",
"both-pending",
"nickname-rejected",
"photo-rejected",
"deadline-near",
"completed",
"timed-out",
]);
export const PlannerSuggestedToolSchema = z.enum([
"validate-nickname",
"check-onboarding-progress",
"lookup-community-rule",
"get-room-context",
]);
export const PlannerToneSchema = z.enum([
"friendly",
"concise",
"encouraging",
"neutral",
"apologetic",
]);
export const PlannerOutputSchema = z.object({
approach: PlannerApproachSchema,
targetPhase: PlannerTargetPhaseSchema.optional(),
mustInclude: z.array(z.string().min(1).max(80)).max(5).default([]),
mustAvoid: z.array(z.string().min(1).max(80)).max(5).default([]),
suggestedTools: z.array(PlannerSuggestedToolSchema).max(3).default([]),
tone: PlannerToneSchema.default("friendly"),
targetLengthChars: z.number().int().min(20).max(500).default(150),
rationale: z.string().max(200),
});
export type PlannerOutput = z.infer<typeof PlannerOutputSchema>;
@@ -0,0 +1,219 @@
import { describe, expect, it } from "vitest";
import { z } from "zod";
import { runWithTools } from "./tool-loop.js";
import type { Tool } from "@tools/tool";
describe("runWithTools", () => {
it("executes a tool call and returns the model's final answer", async () => {
const calls: Array<{
content: string;
messages: Array<{ role: string; content: string }>;
options?: Record<string, unknown> | undefined;
}> = [];
const llmClientStub = {
chatCompletion: (
messages: Array<{ role: string; content: string }>,
options?: Record<string, unknown>,
) => {
calls.push({
content: messages[messages.length - 1]?.content ?? "",
messages,
options,
});
if (calls.length === 1) {
return Promise.resolve({
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
endpoint: "mock://llm",
provider: "llama",
raw: {
choices: [
{
message: {
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
},
],
},
message: {
role: "assistant",
content:
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
},
});
}
return Promise.resolve({
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
endpoint: "mock://llm",
provider: "llama",
raw: {
choices: [
{
message: {
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
},
},
],
},
message: {
role: "assistant",
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
},
});
},
};
const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = {
name: "validate-nickname",
description: "닉네임 형식 검사",
argsSchema: z.object({
nickname: z.string(),
}),
resultSchema: z.object({
valid: z.boolean(),
issues: z.array(z.string()),
}),
execute: () =>
Promise.resolve({
valid: false,
issues: ["티어는 '/'로 구분해서 넣어주세요."],
}),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [
{
role: "system",
content: "도구를 필요할 때만 사용해.",
},
{
role: "user",
content: "닉네임이 왜 안 돼요?",
},
],
toolContext: {},
tools: [tool],
});
expect(result).toContain("프리리/95/여/D");
expect(calls[0]?.options?.tools).toHaveLength(1);
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
});
it("preserves native tool call responses for follow-up turns", async () => {
const seenMessages: Array<Array<Record<string, unknown>>> = [];
const llmClientStub = {
chatCompletion: (messages: Array<Record<string, unknown>>) => {
seenMessages.push(messages);
if (seenMessages.length === 1) {
return Promise.resolve({
content: "",
endpoint: "mock://cliproxy",
provider: "cliproxy",
raw: {
choices: [
{
message: {
content: null,
tool_calls: [
{
id: "call_1",
type: "function",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
},
},
],
},
message: {
role: "assistant",
content: "",
tool_calls: [
{
id: "call_1",
type: "function",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
},
});
}
return Promise.resolve({
content: "나이 기준은 27세 이상이에요.",
endpoint: "mock://cliproxy",
provider: "cliproxy",
raw: {
choices: [
{
message: {
content: "나이 기준은 27세 이상이에요.",
},
},
],
},
message: {
role: "assistant",
content: "나이 기준은 27세 이상이에요.",
},
});
},
};
const tool: Tool<{ topic: string }, { topic: string; text: string }> = {
name: "lookup-community-rule",
description: "커뮤니티 규칙 조회",
argsSchema: z.object({
topic: z.string(),
}),
resultSchema: z.object({
topic: z.string(),
text: z.string(),
}),
execute: () =>
Promise.resolve({
topic: "age",
text: "나이 기준은 27세 이상이에요.",
}),
};
const result = await runWithTools({
llmClient: llmClientStub as never,
messages: [
{
role: "system",
content: "필요할 때만 도구를 사용해.",
},
{
role: "user",
content: "나이 기준이 뭐예요?",
},
],
toolContext: {},
tools: [tool],
});
expect(result).toBe("나이 기준은 27세 이상이에요.");
expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true);
expect(
seenMessages[1]?.some(
(message) => Array.isArray(message.tool_calls) && message.role === "assistant",
),
).toBe(true);
});
});
@@ -0,0 +1,89 @@
import { withToolSpan } from "@obs/spans";
import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
import type { Tool, ToolContext } from "@tools/tool";
export interface ToolLoopOptions {
deadlineMs?: number | undefined;
llmClient: LlmClient;
maxHops?: number | undefined;
maxTokens?: number | undefined;
messages: ChatMessage[];
modelOverride?: string | undefined;
role?: ChatCompletionOptions["role"];
signal?: AbortSignal | undefined;
temperature?: number | undefined;
tier?: ChatCompletionOptions["tier"];
toolContext: ToolContext;
tools: readonly Tool<unknown, unknown>[];
}
export async function runWithTools(
options: ToolLoopOptions,
): Promise<string | null> {
const maxHops = options.maxHops ?? 2;
const messages = options.messages.map((message) => ({ ...message }));
let allowNativeTools = true;
for (let hop = 0; hop <= maxHops; hop++) {
const response = await options.llmClient.chatCompletion(messages, {
deadlineMs: options.deadlineMs,
maxTokens: options.maxTokens,
modelOverride: options.modelOverride,
role: options.role,
signal: options.signal,
temperature: options.temperature,
tier: options.tier,
tools: allowNativeTools ? options.tools : undefined,
});
if (!response) {
return null;
}
const toolCalls = parseToolCalls(response.message);
if (toolCalls.length === 0) {
return response.content;
}
if (hop === maxHops) {
messages.push(response.message);
messages.push({
role: "user",
content:
"도구 호출은 여기까지 하고, 지금까지 얻은 정보만으로 한국어 문장으로 최종 답변해줘.",
});
allowNativeTools = false;
continue;
}
messages.push(response.message);
for (const call of toolCalls) {
const tool = options.tools.find((candidate) => candidate.name === call.name);
if (!tool) {
throw new Error(`Tool not registered for call: ${call.name}`);
}
const validated = tool.argsSchema.parse(call.arguments);
const result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
messages.push({
role: "tool",
name: tool.name,
tool_call_id: call.id,
content: JSON.stringify(result),
});
}
}
return null;
}
@@ -0,0 +1,67 @@
import { describe, expect, it } from "vitest";
import { validateOnboardingResponse } from "./validator.js";
import type { DerivedState } from "../state/onboarding-state.js";
function makeDerived(
overrides?: Partial<DerivedState>,
): DerivedState {
return {
phase: { kind: "photo-pending" },
intent: { kind: "photo-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 256,
allowShortResponse: false,
tier: 0,
...overrides,
};
}
describe("validateOnboardingResponse", () => {
it("rejects discord guidance before onboarding is completed", () => {
const result = validateOnboardingResponse(
"디스코드 서버에 들어가시면 돼요.",
makeDerived(),
);
expect(result.status).toBe("reject");
expect(result.reasons).toContain("stateConsistency");
});
it("rejects invalid nickname examples", () => {
const result = validateOnboardingResponse(
"홍길동/95/남/Z로 바꾸시면 돼요.",
makeDerived({
phase: { kind: "nickname-rejected", current: "홍길동/95/남/골드", issues: [] },
intent: { kind: "nickname-question" },
}),
);
expect(result.status).toBe("reject");
expect(result.reasons).toContain("nicknameExampleConsistency");
});
it("soft-warns on repeated phrases", () => {
const result = validateOnboardingResponse(
"하트 인증 해주세요 하트 인증 해주세요 하트 인증 해주세요",
makeDerived(),
{ allowShortResponse: true },
);
expect(result.status).toBe("soft-warn");
expect(result.reasons).toContain("phraseRepetition");
});
it("passes short status responses when allowShortResponse is enabled", () => {
const result = validateOnboardingResponse(
"이제 방 하트 인증만 남았어요.",
makeDerived({
intent: { kind: "status-question" },
}),
{ allowShortResponse: true },
);
expect(result.status).toBe("pass");
});
});
+180
View File
@@ -0,0 +1,180 @@
import { trace } from "@opentelemetry/api";
import { getMaxResponseLength } from "../../../config.js";
import { VALID_TIER_SET } from "../constants.js";
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
import type { DerivedState } from "../state/onboarding-state.js";
const ALLOWED_URL_PATTERNS = [
/discord\.gg\/lolympus/iu,
/open\.kakao\.com\/o\/gOscBAEh/iu,
];
export interface ValidationResult {
status: "pass" | "soft-warn" | "reject";
text: string;
reasons: string[];
}
function postProcessResponse(text: string): string {
let processed = text.trim();
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
processed = processed.replace(
/https?:\/\/[^\s)}\]]+/gu,
(match) => {
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
return match;
}
return "";
},
);
processed = processed.replace(/ +/gu, " ").trim();
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1");
processed = processed.replace(/\*(.+?)\*/gu, "$1");
processed = processed.replace(/__(.+?)__/gu, "$1");
processed = processed.replace(/~~(.+?)~~/gu, "$1");
processed = processed.replace(/^#{1,6}\s+/gmu, "");
processed = processed.replace(/^[-*]\s+/gmu, "");
processed = processed.replace(/`(.+?)`/gu, "$1");
const maxLen = getMaxResponseLength();
if (processed.length > maxLen) {
processed = processed.slice(0, maxLen).trimEnd();
const sentenceEndPattern =
/(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
for (let index = processed.length; index > maxLen * 0.5; index--) {
const sub = processed.slice(0, index);
if (sentenceEndPattern.test(sub)) {
processed = sub;
break;
}
}
}
return processed.trim();
}
function findInvalidNicknameExamples(text: string): string[] {
const matches = [
...text.matchAll(/([^\s/]{1,20})\/(\d{2})\/(남|여)\/([A-Z]{1,2})/gu),
];
return matches.flatMap((match) => {
const tier = match[4];
if (!tier || VALID_TIER_SET.has(tier)) {
return [];
}
return [tier];
});
}
function hasIntentKeywordMatch(text: string, derived: DerivedState): boolean {
const keywords = EXPECTED_KEYWORDS[derived.intent.kind];
if (keywords.length === 0) {
return true;
}
const lower = text.toLowerCase();
return keywords.some((keyword) => lower.includes(keyword.toLowerCase()));
}
function hasRepeatedPhrase(text: string): boolean {
const words = text.split(/\s+/u);
if (words.length < 6) {
return false;
}
const phrases = new Map<string, number>();
for (let index = 0; index < words.length - 1; index++) {
const bigram = `${words[index]} ${words[index + 1]}`;
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
}
return [...phrases.values()].some((count) => count >= 3);
}
function markValidation(result: ValidationResult): ValidationResult {
const activeSpan = trace.getActiveSpan();
activeSpan?.setAttributes({
"agent.validator.status": result.status,
"agent.validator.warnings": result.reasons,
});
return result;
}
export function validateOnboardingResponse(
draft: string,
derived: DerivedState,
options?: { allowShortResponse?: boolean | undefined },
): ValidationResult {
const text = postProcessResponse(draft);
const rejectReasons: string[] = [];
const softWarnReasons: string[] = [];
if (!text) {
rejectReasons.push("empty");
}
if (text.includes("[NO_RESPONSE]")) {
return markValidation({
status: "pass",
text: "[NO_RESPONSE]",
reasons: [],
});
}
if (!options?.allowShortResponse && text.length < 10) {
rejectReasons.push("minLength");
}
const mentionsDiscord = /(디스코드|discord|디코)/iu.test(text);
const advancesDiscordStep = /(서버|프로필|권한|입장)/iu.test(text);
const defersDiscordStep =
/(먼저|완료.*후|완료하신 뒤|완료하신 후|모두 완료|완료되면)/u.test(text);
if (
mentionsDiscord &&
derived.phase.kind !== "completed" &&
advancesDiscordStep &&
!defersDiscordStep
) {
rejectReasons.push("stateConsistency");
}
if (findInvalidNicknameExamples(text).length > 0) {
rejectReasons.push("nicknameExampleConsistency");
}
if (!hasIntentKeywordMatch(text, derived)) {
softWarnReasons.push("intentAlignment");
}
if (hasRepeatedPhrase(text)) {
softWarnReasons.push("phraseRepetition");
}
if (rejectReasons.length > 0) {
return markValidation({
status: "reject",
text,
reasons: [...rejectReasons, ...softWarnReasons],
});
}
if (softWarnReasons.length > 0) {
return markValidation({
status: "soft-warn",
text,
reasons: softWarnReasons,
});
}
return markValidation({
status: "pass",
text,
reasons: [],
});
}
+100 -91
View File
@@ -1,11 +1,17 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from "@agent/types";
import { getConversationSummaryKey } from "@conv/conversation-store";
import type { GarnetClient } from "@common/garnet";
import { getRequestLogger } from "@obs/context";
import { getContext, getRequestLogger } from "@obs/context";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { RoomContextService } from "@room/room-context-service";
import type { Skill } from "@skills/skill";
import {
type ShadowEvalPublisher,
ShadowEvalBus,
} from "@eval/shadow-bus";
import { createBuiltInToolRegistry } from "@tools/builtins/index";
import type { Tool, ToolContext } from "@tools/tool";
import { PromptBuilder } from "./prompts/builder.js";
import {
getAcknowledgementMessage,
@@ -23,103 +29,40 @@ import {
getRemainingStatusMessage,
getTimeoutWarningMessage,
} from "./onboarding-templates.js";
import { getOnboardingSessionKeyPrefix, getMaxResponseLength, getLlmTemperature } from "../../config.js";
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from "../../config.js";
import { OnboardingBlackboardStore } from "./state/blackboard.js";
import { deriveState } from "./state/derive.js";
import { OnboardingIntentClassifier } from "./state/classifier.js";
import type { DerivedState, Intent } from "./state/onboarding-state.js";
import { OnboardingOrchestrator } from "./runtime/orchestrator.js";
import { validateOnboardingResponse } from "./runtime/validator.js";
import type { OnboardingSession, PromptContext } from "./types.js";
const ALLOWED_URL_PATTERNS = [
/discord\.gg\/lolympus/u,
/open\.kakao\.com\/o\/gOscBAEh/u,
];
type LlmTextResult =
| { status: "success"; text: string }
| { status: "no_response" }
| { status: "failed" };
function postProcessResponse(text: string): string {
let processed = text.trim();
// Remove <think> blocks (Qwen thinking mode artifacts)
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
// Strip URLs not in allowlist
processed = processed.replace(
/https?:\/\/[^\s)}\]]+/gu,
(match) => {
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
return match;
}
return "";
},
);
// Clean up double spaces from URL removal
processed = processed.replace(/ +/gu, " ").trim();
// Strip markdown formatting (KakaoTalk is plaintext)
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1"); // **bold**
processed = processed.replace(/\*(.+?)\*/gu, "$1"); // *italic*
processed = processed.replace(/__(.+?)__/gu, "$1"); // __underline__
processed = processed.replace(/~~(.+?)~~/gu, "$1"); // ~~strike~~
processed = processed.replace(/^#{1,6}\s+/gmu, ""); // ## headings
processed = processed.replace(/^[-*]\s+/gmu, ""); // - or * list items
processed = processed.replace(/`(.+?)`/gu, "$1"); // `inline code`
// Truncate to max length with Korean sentence boundary awareness
const maxLen = getMaxResponseLength();
if (processed.length > maxLen) {
processed = processed.slice(0, maxLen).trimEnd();
// Search backwards for a Korean sentence-ending pattern
const sentenceEndPattern = /(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
// Try progressively shorter substrings to find a sentence boundary
for (let i = processed.length; i > maxLen * 0.5; i--) {
const sub = processed.slice(0, i);
if (sentenceEndPattern.test(sub)) {
processed = sub;
break;
}
}
}
return processed.trim();
}
function isLowQualityResponse(
text: string,
options?: { allowShortResponse?: boolean },
): boolean {
if (!options?.allowShortResponse && text.length < 10) return true;
// Check for excessive repetition (same phrase 3+ times)
const words = text.split(/\s+/u);
if (words.length >= 6) {
const phrases = new Map<string, number>();
for (let i = 0; i < words.length - 1; i++) {
const bigram = `${words[i]} ${words[i + 1]}`;
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
}
for (const count of phrases.values()) {
if (count >= 3) return true;
}
}
return false;
}
export class OnboardingSkill implements Skill {
readonly name = "onboarding";
private readonly classifier: OnboardingIntentClassifier;
private readonly blackboardStore: OnboardingBlackboardStore;
private readonly orchestrator: OnboardingOrchestrator;
private readonly shadowEvalPublisher: ShadowEvalPublisher;
private readonly tools: readonly Tool<unknown, unknown>[];
constructor(
private readonly garnetClient: GarnetClient,
private readonly llmClient: LlmClient,
private readonly roomContextService?: RoomContextService,
shadowEvalPublisher?: ShadowEvalPublisher,
) {
this.classifier = new OnboardingIntentClassifier(llmClient);
this.blackboardStore = new OnboardingBlackboardStore(garnetClient);
this.orchestrator = new OnboardingOrchestrator(llmClient);
this.shadowEvalPublisher =
shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
this.tools = createBuiltInToolRegistry().list();
}
canHandle(request: AgentRequest): boolean {
@@ -185,6 +128,15 @@ export class OnboardingSkill implements Skill {
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, "[OnboardingSkill] Failed to update blackboard");
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
request,
response,
});
}
return response;
@@ -212,11 +164,15 @@ export class OnboardingSkill implements Skill {
derived,
);
return this.generateLlmText(messages, {
return this.generateLlmText(messages, derived, {
allowShortResponse: derived.allowShortResponse,
maxTokens: derived.suggestedMaxTokens,
retryOnLowQuality:
request.trigger === "user-message" && !derived.allowShortResponse,
request.trigger === "user-message" &&
!derived.allowShortResponse &&
derived.responseMode !== "llm-with-tools",
toolContext: this.buildToolContext(request, session),
tools: this.selectToolsForDerived(derived),
});
}
@@ -456,11 +412,14 @@ export class OnboardingSkill implements Skill {
private async generateLlmText(
messages: ChatMessage[],
derived: DerivedState,
options?: {
allowShortResponse?: boolean;
maxTokens?: number;
retryOnLowQuality?: boolean;
temperature?: number;
toolContext?: ToolContext | undefined;
tools?: readonly Tool<unknown, unknown>[] | undefined;
},
): Promise<LlmTextResult> {
if (!this.llmClient.hasEndpoints()) {
@@ -509,39 +468,79 @@ export class OnboardingSkill implements Skill {
requestOptions.temperature = attempt.temperature;
}
const llmResult = await this.llmClient.chatCompletion(
const toolContext = options?.toolContext ?? {};
const llmText = await this.orchestrator.run(
derived,
messages,
requestOptions,
{
maxTokens: requestOptions.maxTokens,
temperature: requestOptions.temperature,
toolContext,
tools: options?.tools,
},
);
if (!llmResult) {
if (!llmText) {
continue;
}
const processed = postProcessResponse(llmResult.content);
if (!processed) {
continue;
}
if (processed.includes("[NO_RESPONSE]")) {
const validation = validateOnboardingResponse(
llmText,
derived,
{
allowShortResponse: options?.allowShortResponse,
},
);
if (validation.text.includes("[NO_RESPONSE]")) {
return { status: "no_response" };
}
const qualityOptions =
options?.allowShortResponse !== undefined
? { allowShortResponse: options.allowShortResponse }
: undefined;
if (isLowQualityResponse(processed, qualityOptions)) {
if (validation.status === "reject") {
continue;
}
return {
status: "success",
text: processed,
text: validation.text,
};
}
return { status: "failed" };
}
private buildToolContext(
request: AgentRequest,
session: OnboardingSession | null,
): ToolContext {
return {
chatroomId: request.chatroomId,
userId: request.userId,
session,
garnetClient: this.garnetClient,
roomContextService: this.roomContextService,
};
}
private selectToolsForDerived(
derived: DerivedState,
): readonly Tool<unknown, unknown>[] {
if (derived.phase.kind === "nickname-rejected") {
return this.tools.filter((tool) =>
tool.name === "validate-nickname" ||
tool.name === "lookup-community-rule" ||
tool.name === "check-onboarding-progress",
);
}
if (derived.intent.kind === "status-question") {
return this.tools.filter((tool) =>
tool.name === "check-onboarding-progress",
);
}
return this.tools;
}
private makeTemplateResponse(
requestId: string,
text: string,
@@ -574,4 +573,14 @@ export class OnboardingSkill implements Skill {
source: "template",
};
}
private getCurrentLatencyMs(): number | undefined {
const startedAt = getContext()?.startedAt;
const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN;
if (!Number.isFinite(startedTimestamp)) {
return undefined;
}
return Math.max(0, Date.now() - startedTimestamp);
}
}
@@ -1,16 +1,16 @@
import type { Intent } from "./onboarding-state.js";
export const EXPECTED_KEYWORDS: Record<Intent["kind"], string[]> = {
acknowledgement: ["감사", "알겠", "확인"],
"age-question": ["27", "나이", "가입"],
acknowledgement: ["네", "좋아요", "천천히"],
"age-question": ["27", "가입", "가능"],
chitchat: [],
"discord-question": ["디스코드", "디코", "discord"],
greeting: ["안녕", "반갑", "환영"],
"help-how": ["어떻게", "방법", "해"],
"help-where": ["어디", "위치", "찾"],
"nickname-question": ["닉네임", "닉변", "형식", "이름"],
"discord-question": ["디스코드", "권한", "완료"],
greeting: ["반갑", "환영", "어서 오세요"],
"help-how": ["하트", "닉네임", "스크린샷", "형식", "커버 보기"],
"help-where": ["오른쪽 상단", "커버 보기", "프로필"],
"nickname-question": ["닉네임", "형식", "티어", "/"],
"off-topic-to-other": [],
"photo-question": ["하트", "인증", "스크린샷", "프로필"],
"status-question": ["남았", "완료", "진행", "상태"],
"photo-question": ["하트", "스크린샷", "프로필", "커버 보기"],
"status-question": ["남았", "완료", "아직", "끝"],
unknown: [],
};
+2 -2
View File
@@ -1,4 +1,4 @@
import { getDeadlineNearThresholdMin } from "../../../config.js";
import { DEADLINE_NEAR_THRESHOLD_MIN } from "../constants.js";
import { analyzeNicknameIssues } from "../onboarding-templates.js";
import type { OnboardingSession } from "../types.js";
import type { OnboardingPhase, RemainingStep } from "./onboarding-state.js";
@@ -30,7 +30,7 @@ function isDeadlineNear(session: OnboardingSession | null): boolean {
return false;
}
return deadlineAt - Date.now() <= getDeadlineNearThresholdMin() * 60_000;
return deadlineAt - Date.now() <= DEADLINE_NEAR_THRESHOLD_MIN * 60_000;
}
export function derivePhaseFromSession(
@@ -0,0 +1,64 @@
import { z } from "zod";
import { getOnboardingSessionKeyPrefix } from "../../config.js";
import type { OnboardingSession } from "@skills/onboarding/types";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
chatroomId: z.string().min(1),
userId: z.string().min(1),
});
const resultSchema = z.object({
nicknameDone: z.boolean(),
photoDone: z.boolean(),
remainingSteps: z.array(z.enum(["nickname", "photo"])),
deadlineInHours: z.number().nullable(),
});
function getRemainingSteps(session: OnboardingSession | null): Array<"nickname" | "photo"> {
if (!session) {
return ["nickname", "photo"];
}
const remaining: Array<"nickname" | "photo"> = [];
if (!session.nicknameVerified) {
remaining.push("nickname");
}
if (!session.photoVerified) {
remaining.push("photo");
}
return remaining;
}
export const checkOnboardingProgressTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "check-onboarding-progress",
description:
"현재 유저의 온보딩 진행 상태를 읽어 남은 절차와 데드라인까지 남은 시간을 구조화해서 반환한다.",
argsSchema,
resultSchema,
async execute(args, ctx) {
const session =
ctx.session && ctx.chatroomId === args.chatroomId && ctx.userId === args.userId
? ctx.session
: ctx.garnetClient
? await ctx.garnetClient.getJson<OnboardingSession>(
`${getOnboardingSessionKeyPrefix()}:onboarding:${args.chatroomId}:${args.userId}`,
)
: null;
const deadlineAt = session?.deadlineAt ? Date.parse(session.deadlineAt) : NaN;
const deadlineInHours = Number.isFinite(deadlineAt)
? Math.max(0, (deadlineAt - Date.now()) / 3_600_000)
: null;
return {
nicknameDone: session?.nicknameVerified ?? false,
photoDone: session?.photoVerified ?? false,
remainingSteps: getRemainingSteps(session),
deadlineInHours,
};
},
};
+48
View File
@@ -0,0 +1,48 @@
import { z } from "zod";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
chatroomId: z.string().min(1),
limit: z.number().int().min(1).max(20).optional(),
});
const roomMessageSchema = z.object({
senderId: z.string(),
nickname: z.string(),
message: z.string(),
timestamp: z.string(),
});
const resultSchema = z.object({
messages: z.array(roomMessageSchema),
});
export const getRoomContextTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "get-room-context",
description:
"최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.",
argsSchema,
resultSchema,
async execute(args, ctx) {
if (!ctx.roomContextService) {
return { messages: [] };
}
const messages = await ctx.roomContextService.getRecentMessages(
args.chatroomId,
args.limit,
);
return {
messages: messages.map((message) => ({
senderId: message.senderId,
nickname: message.nickname,
message: message.message,
timestamp: message.timestamp.toISOString(),
})),
};
},
};
+20
View File
@@ -0,0 +1,20 @@
import { ToolRegistry } from "../registry.js";
import { checkOnboardingProgressTool } from "./check-onboarding-progress.js";
import { getRoomContextTool } from "./get-room-context.js";
import { lookupCommunityRuleTool } from "./lookup-community-rule.js";
import { validateNicknameTool } from "./validate-nickname.js";
export function createBuiltInToolRegistry(): ToolRegistry {
return new ToolRegistry()
.register(validateNicknameTool)
.register(checkOnboardingProgressTool)
.register(lookupCommunityRuleTool)
.register(getRoomContextTool);
}
export {
checkOnboardingProgressTool,
getRoomContextTool,
lookupCommunityRuleTool,
validateNicknameTool,
};
@@ -0,0 +1,54 @@
import { z } from "zod";
import {
DEADLINE_NEAR_THRESHOLD_MIN,
DISCORD_GUIDE_TEXT,
HEART_RULE_TEXT,
MIN_AGE,
NICKNAME_FORMAT_TEXT,
TIER_CODES_TEXT,
} from "@skills/onboarding/constants";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
topic: z.enum([
"age",
"tier-codes",
"discord-link",
"heart-rule",
"nickname-format",
"deadline",
]),
});
const resultSchema = z.object({
topic: argsSchema.shape.topic,
text: z.string(),
source: z.literal("constants"),
});
const TOPIC_TEXT: Record<z.infer<typeof argsSchema>["topic"], string> = {
age: `가입 가능 연령은 ${MIN_AGE}세 이상이에요.`,
"tier-codes": `사용 가능한 티어 코드는 ${TIER_CODES_TEXT}예요.`,
"discord-link": DISCORD_GUIDE_TEXT,
"heart-rule": HEART_RULE_TEXT,
"nickname-format": NICKNAME_FORMAT_TEXT,
deadline: `데드라인이 ${DEADLINE_NEAR_THRESHOLD_MIN}분 이하로 남으면 마감이 임박한 상태로 안내해요.`,
};
export const lookupCommunityRuleTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "lookup-community-rule",
description:
"나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 상수에서 조회한다.",
argsSchema,
resultSchema,
execute(args) {
return Promise.resolve({
topic: args.topic,
text: TOPIC_TEXT[args.topic],
source: "constants",
});
},
};
+52
View File
@@ -0,0 +1,52 @@
import { z } from "zod";
import { analyzeNicknameIssues } from "@skills/onboarding/onboarding-templates";
import type { Tool } from "../tool.js";
const argsSchema = z.object({
nickname: z.string().min(1),
});
const nicknameIssueSchema = z.object({
kind: z.enum([
"missing-tier",
"invalid-age",
"missing-gender",
"wrong-separator",
"unknown-tier",
"other",
]),
message: z.string(),
suggestion: z.string().optional(),
});
const resultSchema = z.object({
valid: z.boolean(),
issues: z.array(nicknameIssueSchema),
normalized: z.string().optional(),
});
export const validateNicknameTool: Tool<
z.infer<typeof argsSchema>,
z.infer<typeof resultSchema>
> = {
name: "validate-nickname",
description:
"새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.",
argsSchema,
resultSchema,
execute(args) {
const normalized = args.nickname.trim();
const issues = analyzeNicknameIssues(normalized);
const result: z.infer<typeof resultSchema> = {
valid: issues.length === 0,
issues,
};
if (issues.length === 0) {
result.normalized = normalized;
}
return Promise.resolve(result);
},
};
+8
View File
@@ -0,0 +1,8 @@
export { ToolRegistry } from "./registry.js";
export { parseToolCalls } from "./parser.js";
export {
serializeToolsForHermes,
serializeToolsForOpenAi,
} from "./serializer.js";
export type { OpenAiToolDefinition } from "./serializer.js";
export type { Tool, ToolContext } from "./tool.js";
+70
View File
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import { parseToolCalls } from "./parser.js";
describe("parseToolCalls", () => {
it("parses Hermes tool call blocks", () => {
const result = parseToolCalls(
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여/D"}}</tool_call>',
);
expect(result).toEqual([
{
name: "validate-nickname",
arguments: {
nickname: "프리리/95/여/D",
},
},
]);
});
it("parses OpenAI tool_calls arrays", () => {
const result = parseToolCalls({
tool_calls: [
{
id: "call_1",
function: {
name: "lookup-community-rule",
arguments: '{"topic":"age"}',
},
},
],
});
expect(result).toEqual([
{
id: "call_1",
name: "lookup-community-rule",
arguments: {
topic: "age",
},
},
]);
});
it("parses tool calls nested in choices[0].message", () => {
const result = parseToolCalls({
choices: [
{
message: {
tool_calls: [
{
id: "call_2",
function: {
name: "check-onboarding-progress",
arguments: '{"chatroomId":"room-1","userId":"user-1"}',
},
},
],
},
},
],
});
expect(result[0]?.name).toBe("check-onboarding-progress");
expect(result[0]?.arguments).toEqual({
chatroomId: "room-1",
userId: "user-1",
});
});
});
+136
View File
@@ -0,0 +1,136 @@
export interface ParsedToolCall {
id?: string | undefined;
name: string;
arguments: Record<string, unknown>;
}
interface OpenAiToolCallShape {
id?: string;
function?: {
name?: string;
arguments?: string;
};
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function parseJsonArguments(raw: unknown): Record<string, unknown> | null {
if (typeof raw === "string") {
try {
const parsed = JSON.parse(raw) as unknown;
return asRecord(parsed);
} catch {
return null;
}
}
if (typeof raw === "object" && raw !== null) {
return raw as Record<string, unknown>;
}
return null;
}
function parseHermesCalls(content: string): ParsedToolCall[] {
const matches = [...content.matchAll(/<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/gu)];
return matches.flatMap((match) => {
const payload = match[1];
if (!payload) {
return [];
}
try {
const parsed = JSON.parse(payload) as
| {
id?: string;
name?: string;
arguments?: unknown;
}
| Array<{
id?: string;
name?: string;
arguments?: unknown;
}>;
const items = Array.isArray(parsed) ? parsed : [parsed];
return items.flatMap((item) => {
const args = parseJsonArguments(item.arguments);
if (!item.name || !args) {
return [];
}
return [
{
id: item.id,
name: item.name,
arguments: args,
},
];
});
} catch {
return [];
}
});
}
function parseOpenAiCalls(value: unknown): ParsedToolCall[] {
const record = asRecord(value);
const choices = Array.isArray(record["choices"])
? (record["choices"] as unknown[])
: [];
const firstChoice = choices[0];
const candidates = [
record["tool_calls"],
asRecord(record["message"])["tool_calls"],
asRecord(asRecord(firstChoice)["message"])["tool_calls"],
];
const toolCallsCandidate = candidates.find(Array.isArray);
if (!Array.isArray(toolCallsCandidate)) {
return [];
}
return toolCallsCandidate.flatMap((item) => {
const call = item as OpenAiToolCallShape;
const args = parseJsonArguments(call.function?.arguments);
const name = call.function?.name;
if (!name || !args) {
return [];
}
return [
{
id: call.id,
name,
arguments: args,
},
];
});
}
export function parseToolCalls(value: unknown): ParsedToolCall[] {
if (typeof value === "string") {
return parseHermesCalls(value);
}
const record = asRecord(value);
const content =
typeof record["content"] === "string"
? record["content"]
: typeof asRecord(record["message"])["content"] === "string"
? (asRecord(record["message"])["content"] as string)
: "";
const parsedOpenAiCalls = parseOpenAiCalls(value);
if (parsedOpenAiCalls.length > 0) {
return parsedOpenAiCalls;
}
return content ? parseHermesCalls(content) : [];
}
+32
View File
@@ -0,0 +1,32 @@
import type { Tool } from "./tool.js";
export class ToolRegistry {
private readonly tools = new Map<string, Tool<unknown, unknown>>();
register<A, R>(tool: Tool<A, R>): this {
this.tools.set(tool.name, tool as Tool<unknown, unknown>);
return this;
}
get(name: string): Tool<unknown, unknown> {
const tool = this.tools.get(name);
if (!tool) {
throw new Error(`Unknown tool: ${name}`);
}
return tool;
}
has(name: string): boolean {
return this.tools.has(name);
}
list(): Tool<unknown, unknown>[] {
return [...this.tools.values()];
}
pick(names: readonly string[]): Tool<unknown, unknown>[] {
return names
.filter((name) => this.tools.has(name))
.map((name) => this.get(name));
}
}
+42
View File
@@ -0,0 +1,42 @@
import { z } from "zod";
import type { Tool } from "./tool.js";
export interface OpenAiToolDefinition {
type: "function";
function: {
name: string;
description: string;
parameters: unknown;
};
}
function toToolDescriptor(tool: Tool<unknown, unknown>): {
name: string;
description: string;
parameters: unknown;
} {
return {
name: tool.name,
description: tool.description,
parameters: z.toJSONSchema(tool.argsSchema),
};
}
export function serializeToolsForHermes(
tools: readonly Tool<unknown, unknown>[],
): string {
const descriptors = tools.map(toToolDescriptor);
return `<tools>\n${JSON.stringify(descriptors, null, 2)}\n</tools>`;
}
export function serializeToolsForOpenAi(
tools: readonly Tool<unknown, unknown>[],
): OpenAiToolDefinition[] {
return tools.map((tool) => {
const descriptor = toToolDescriptor(tool);
return {
type: "function",
function: descriptor,
};
});
}
+20
View File
@@ -0,0 +1,20 @@
import type { GarnetClient } from "@common/garnet";
import type { RoomContextService } from "@room/room-context-service";
import type { ZodType } from "zod";
import type { OnboardingSession } from "@skills/onboarding/types";
export interface ToolContext {
chatroomId?: string | undefined;
userId?: string | undefined;
session?: OnboardingSession | null | undefined;
garnetClient?: GarnetClient | undefined;
roomContextService?: RoomContextService | undefined;
}
export interface Tool<A, R> {
name: string;
description: string;
argsSchema: ZodType<A>;
resultSchema: ZodType<R>;
execute(args: A, ctx: ToolContext): Promise<R>;
}
+207
View File
@@ -0,0 +1,207 @@
import { loadEnvironmentFromNodeEnv } from "../common/env-loader.js";
import {
getOpenRouterApiUrl,
getOpenRouterEnabled,
} from "../config.js";
loadEnvironmentFromNodeEnv();
const baseServiceName = process.env["SERVICE_NAME"]?.trim() || "lolympus-agent";
process.env["SERVICE_NAME"] = `${baseServiceName}-worker`;
const { getLogger, initLogger } = await import("../observability/logger.js");
initLogger();
const logger = getLogger();
const { initTracing, shutdownTracing } = await import(
"../observability/tracing.js"
);
initTracing();
const { GarnetClient } = await import("../common/garnet.js");
const { NatsClient } = await import("../common/nats.js");
const { LlmClient } = await import("../llm/index.js");
const { getRawSecret } = await import("../common/secret-manager.js");
const { ConversationStore } = await import("../conversation/index.js");
const { ShadowWorker } = await import("../eval/shadow-worker.js");
const { AlertService } = await import("../observability/alert.js");
function firstNonEmpty(
...values: Array<string | null | undefined>
): string | undefined {
for (const value of values) {
if (value?.trim()) {
return value.trim();
}
}
return undefined;
}
function parseEndpointList(
...values: Array<string | null | undefined>
): string[] {
const items: string[] = [];
for (const value of values) {
if (!value?.trim()) {
continue;
}
for (const item of value.split(/[\n,]/u)) {
const trimmed = item.trim();
if (trimmed) {
items.push(trimmed);
}
}
}
return [...new Set(items)];
}
function toProviderEndpoints(
urls: string[],
provider?: "llama" | "cliproxy" | "openrouter",
): Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }> {
if (!provider) {
return urls;
}
return urls.map((url) => ({ url, provider }));
}
function installShutdownHooks(
shutdown: (reason: string, error?: unknown) => Promise<void>,
): void {
let shuttingDown = false;
const runShutdown = async (
reason: string,
error?: unknown,
): Promise<void> => {
if (shuttingDown) {
return;
}
shuttingDown = true;
if (error) {
logger.error({ err: error }, `[Background] ${reason}`);
} else {
logger.info(`[Background] ${reason}`);
}
try {
await shutdown(reason, error);
} catch (shutdownError) {
logger.error({ err: shutdownError }, "[Background] Cleanup failed");
} finally {
process.exit(error ? 1 : 0);
}
};
for (const signal of ["SIGINT", "SIGTERM"] as const) {
process.once(signal, () => {
void runShutdown(signal);
});
}
process.once("uncaughtException", (error) => {
void runShutdown("uncaughtException", error);
});
process.once("unhandledRejection", (reason) => {
void runShutdown("unhandledRejection", reason);
});
}
async function bootstrap(): Promise<void> {
const serviceName = process.env["SERVICE_NAME"] ?? "lolympus-agent-worker";
const garnetClient = new GarnetClient();
await garnetClient.connect();
const natsClient = new NatsClient();
await natsClient.connect();
const cliproxyEndpoints = parseEndpointList(
getRawSecret("CLI_PROXY_API_URLS"),
process.env["CLI_PROXY_API_URLS"],
);
const generalLlmEndpoints = parseEndpointList(
getRawSecret("AGENT_LLM_API_URLS"),
process.env["AGENT_LLM_API_URLS"],
getRawSecret("LLAMA_SERVER_API_URL"),
process.env["LLAMA_SERVER_API_URL"],
);
const openRouterEndpoints = getOpenRouterEnabled()
? parseEndpointList(
getRawSecret("OPENROUTER_API_URL"),
process.env["OPENROUTER_API_URL"],
getOpenRouterApiUrl(),
)
: [];
const llmApiKey = firstNonEmpty(
getRawSecret("AGENT_LLM_API_KEY"),
process.env["AGENT_LLM_API_KEY"],
getRawSecret("LLAMA_SERVER_API_KEY"),
process.env["LLAMA_SERVER_API_KEY"],
);
const providerApiKeys = Object.fromEntries(
[
[
"cliproxy",
firstNonEmpty(
getRawSecret("CLI_PROXY_API_KEY"),
process.env["CLI_PROXY_API_KEY"],
),
],
["llama", llmApiKey],
[
"openrouter",
firstNonEmpty(
getRawSecret("OPENROUTER_API_KEY"),
process.env["OPENROUTER_API_KEY"],
),
],
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
) as Partial<Record<"cliproxy" | "llama" | "openrouter", string>>;
const llmClient = new LlmClient({
endpoints: [
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
...toProviderEndpoints(generalLlmEndpoints, "llama"),
],
garnetClient,
apiKey: llmApiKey,
model: firstNonEmpty(
getRawSecret("AGENT_LLM_MODEL"),
process.env["AGENT_LLM_MODEL"],
),
providerApiKeys,
});
const conversationStore = new ConversationStore(garnetClient);
const alertService = new AlertService(garnetClient, natsClient);
const worker = new ShadowWorker(
garnetClient,
llmClient,
conversationStore,
alertService,
);
installShutdownHooks(async () => {
await worker.stop();
await natsClient.close();
await garnetClient.close();
await shutdownTracing();
});
await worker.start();
logger.info({ serviceName }, "✓ Background worker bootstrapped");
}
try {
await bootstrap();
} catch (error) {
logger.error({ err: error }, "[Background] Fatal startup error");
await shutdownTracing();
process.exitCode = 1;
}
+19
View File
@@ -0,0 +1,19 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"paths": {
"@agent": ["src/agent/index"],
"@agent/*": ["src/agent/*"],
"@skills/*": ["src/skills/*"],
"@llm": ["src/llm/index"],
"@llm/*": ["src/llm/*"],
"@obs/*": ["src/observability/*"],
"@eval/*": ["src/eval/*"],
"@tools/*": ["src/tools/*"],
"@common/*": ["src/common/*"],
"@conv/*": ["src/conversation/*"],
"@room": ["src/room-context/index"],
"@room/*": ["src/room-context/*"]
}
}
}