Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+166 -17
View File
@@ -1,9 +1,3 @@
function printPhaseNotice(command: string): void {
console.info(
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
);
}
function parsePositiveInteger(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
@@ -13,26 +7,181 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
interface ParsedArgs {
command: string;
flags: Map<string, string | true>;
positionals: string[];
}
function parseArgs(argv: string[]): ParsedArgs {
const [command = "run", ...rest] = argv;
const flags = new Map<string, string | true>();
const positionals: string[] = [];
for (let index = 0; index < rest.length; index++) {
const token = rest[index];
if (!token) {
continue;
}
if (!token.startsWith("--")) {
positionals.push(token);
continue;
}
const withoutPrefix = token.slice(2);
const equalsIndex = withoutPrefix.indexOf("=");
if (equalsIndex >= 0) {
const key = withoutPrefix.slice(0, equalsIndex);
const value = withoutPrefix.slice(equalsIndex + 1);
flags.set(key, value || true);
continue;
}
const next = rest[index + 1];
if (next && !next.startsWith("--")) {
flags.set(withoutPrefix, next);
index += 1;
continue;
}
flags.set(withoutPrefix, true);
}
return {
command,
flags,
positionals,
};
}
function getFlagString(
flags: Map<string, string | true>,
key: string,
): string | undefined {
const value = flags.get(key);
return typeof value === "string" ? value : undefined;
}
function hasFlag(
flags: Map<string, string | true>,
key: string,
): boolean {
return flags.has(key);
}
function printRunSummary(report: {
generatedAt: string;
summary: {
cases: number;
passRate: number;
ruleAverageScore: number;
averageLatencyMs: number;
llmAverageOverall: number | null;
llmJudgedCases: number;
};
baselineDelta?: {
passRate: number | null;
ruleAverageScore: number | null;
llmAverageOverall: number | null;
averageLatencyMs: number | null;
} | undefined;
}, reportPath: string): void {
const llmLine =
report.summary.llmAverageOverall === null
? "n/a"
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
console.info(`[eval] generatedAt=${report.generatedAt}`);
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
console.info(
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
);
console.info(`[eval] llmAverageOverall=${llmLine}`);
if (report.baselineDelta) {
console.info(
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
);
}
console.info(`[eval] report=${reportPath}`);
}
async function main(): Promise<void> {
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
loadEnvironmentFromNodeEnv();
const { getLogger, initLogger } = await import("@obs/logger");
initLogger();
const { captureGoldenSetScaffold } = await import("./capture.js");
const logger = getLogger();
const [command = "run", arg1, arg2] = process.argv.slice(2);
const parsed = parseArgs(process.argv.slice(2));
const { command, flags, positionals } = parsed;
if (command === "capture") {
const outputPath = await captureGoldenSetScaffold({
limit: parsePositiveInteger(arg1),
outputPath: arg2,
});
getLogger().info({ outputPath }, '[eval] "capture" completed');
return;
try {
if (command === "capture") {
const { captureGoldenSetScaffold } = await import("./capture.js");
const outputPath = await captureGoldenSetScaffold({
limit:
parsePositiveInteger(getFlagString(flags, "limit")) ??
parsePositiveInteger(positionals[0]),
outputPath: getFlagString(flags, "output") ?? positionals[1],
});
logger.info({ outputPath }, '[eval] "capture" completed');
return;
}
if (command === "run") {
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
"./runner.js"
);
const { publishEvalReportIfPossible } = await import(
"./runner.js"
);
const setName = getFlagString(flags, "set") ?? "golden";
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
if (hasFlag(flags, "dry-run")) {
const cases = await loadEvalCases(inputPath, { limit });
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
return;
}
const { report, reportPath } = await runOnboardingEval({
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
inputPath,
limit,
outputPath: getFlagString(flags, "output"),
setName,
});
const publishedReport = hasFlag(flags, "no-publish-report")
? false
: await publishEvalReportIfPossible(report);
printRunSummary(report, reportPath);
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
return;
}
if (command === "baseline") {
const { updateEvalBaseline } = await import("./runner.js");
const reportPath = getFlagString(flags, "report") ?? positionals[0];
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
console.info(
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
);
return;
}
throw new Error(`Unknown eval command: ${command}`);
} catch (error) {
logger.error(
{ err: error },
`[eval] "${command}" failed`,
);
process.exitCode = 1;
}
printPhaseNotice(command);
}
void main();
+3
View File
@@ -0,0 +1,3 @@
{"id":"greeting-short","name":"Greeting should stay short and welcoming","input":{"request":{"requestId":"eval_greeting_1","skill":"onboarding","chatroomId":"room_eval_1","userId":"user_eval_1","nickname":"<nick:greeting>","trigger":"user-message","userMessage":"안녕하세요"},"session":null,"conversationHistory":[]},"expected":{"intent":"greeting","responseMode":"llm","mustIncludeAny":["환영","반갑"],"mustNotInclude":["디스코드"]}}
{"id":"status-photo-remaining","name":"Status question should answer only the remaining step","input":{"request":{"requestId":"eval_status_1","skill":"onboarding","chatroomId":"room_eval_2","userId":"user_eval_2","nickname":"프리리/95/여/D","trigger":"user-message","userMessage":"이제 뭐 남았지"},"session":{"chatroomId":"room_eval_2","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/D","nicknameVerified":true,"photoVerified":false,"status":"pending","userId":"user_eval_2"},"conversationHistory":[{"role":"user","content":"안녕하세요","timestamp":"2026-04-16T00:00:00.000Z"},{"role":"assistant","content":"환영합니다! 반갑습니다.","timestamp":"2026-04-16T00:00:01.000Z"}]},"expected":{"intent":"status-question","responseMode":"llm","mustIncludeAny":["남았","하트"],"mustNotInclude":["디스코드"]}}
{"id":"nickname-invalid-tier","name":"Nickname invalid flow should focus on nickname correction","input":{"request":{"requestId":"eval_nick_1","skill":"onboarding","chatroomId":"room_eval_3","userId":"user_eval_3","nickname":"프리리/95/여/다이아","trigger":"nickname-invalid"},"session":{"chatroomId":"room_eval_3","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/다이아","nicknameVerified":false,"photoVerified":false,"status":"pending","userId":"user_eval_3"},"conversationHistory":[]},"expected":{"intent":"nickname-question","responseMode":"llm-with-tools","mustIncludeAny":["닉네임","티어","형식"],"mustNotInclude":["디스코드"]}}
+109
View File
@@ -0,0 +1,109 @@
import { describe, expect, it } from "vitest";
import { LlmClient } from "@llm/llm-client";
import type {
ChatCompletionOptions,
ChatCompletionResult,
ChatMessage,
} from "@llm/types";
import { judgeOnboardingWithLlm } from "./llm-judge.js";
import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js";
class FakeLlmClient extends LlmClient {
constructor(private readonly payload: string | null) {
super({ endpoints: [] });
}
hasEndpoints(): boolean {
return this.payload !== null;
}
chatCompletion(
_messages: ChatMessage[],
_options?: ChatCompletionOptions,
): Promise<ChatCompletionResult | null> {
if (this.payload === null) {
return Promise.resolve(null);
}
return Promise.resolve({
content: this.payload,
endpoint: "http://judge.test/v1/chat/completions",
message: {
role: "assistant",
content: this.payload,
},
provider: "cliproxy",
raw: {},
});
}
}
const evalCase: EvalCaseDefinition = {
id: "case-1",
lineNumber: 1,
input: {
request: {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "이제 뭐 남았지",
},
},
expected: {
intent: "status-question",
responseMode: "llm",
},
};
const derived: EvalDerivedSnapshot = {
allowShortResponse: true,
intent: "status-question",
phase: "photo-pending",
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 128,
tier: 0,
};
describe("judgeOnboardingWithLlm", () => {
it("parses a valid JSON judge response", async () => {
const result = await judgeOnboardingWithLlm(
new FakeLlmClient(
JSON.stringify({
friendliness: 5,
accuracy: 4,
concision: 5,
scope: 4,
rationale: "질문 범위에 맞게 짧고 정확하게 답했다.",
}),
),
evalCase,
"이제 방 하트 인증만 남았어요.",
derived,
);
expect(result.skipped).toBe(false);
if (!result.skipped) {
expect(result.scores.overall).toBe(4.5);
expect(result.provider).toBe("cliproxy");
}
});
it("skips when the judge returns invalid JSON", async () => {
const result = await judgeOnboardingWithLlm(
new FakeLlmClient('{"friendliness":"bad"}'),
evalCase,
"이제 방 하트 인증만 남았어요.",
derived,
);
expect(result.skipped).toBe(true);
if (result.skipped) {
expect(result.reason).toBe("invalid-judge-json");
}
});
});
+134
View File
@@ -0,0 +1,134 @@
import { z } from "zod";
import type { LlmClient } from "@llm/llm-client";
import { getLlmJudgeMaxTokens, getLlmJudgeModel } from "../../config.js";
import type {
EvalCaseDefinition,
EvalDerivedSnapshot,
LlmJudgeResult,
LlmJudgeScores,
} from "../types.js";
const llmJudgeSchema = z.object({
friendliness: z.number().int().min(1).max(5),
accuracy: z.number().int().min(1).max(5),
concision: z.number().int().min(1).max(5),
scope: z.number().int().min(1).max(5),
rationale: z.string().trim().min(1).max(200),
});
function round(value: number): number {
return Number(value.toFixed(3));
}
function buildJudgePrompt(
evalCase: EvalCaseDefinition,
responseText: string,
derived: EvalDerivedSnapshot,
): string {
return [
"다음 온보딩 에이전트 응답을 평가해 주세요.",
"평가 축은 friendliness, accuracy, concision, scope 입니다.",
"각 점수는 1~5의 정수만 사용하세요.",
"scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.",
"",
"[request]",
JSON.stringify(evalCase.input.request),
"",
"[expected]",
JSON.stringify({
intent: evalCase.expected.intent ?? null,
mustIncludeAny: evalCase.expected.mustIncludeAny ?? [],
mustNotInclude: evalCase.expected.mustNotInclude ?? [],
responseMode: evalCase.expected.responseMode ?? null,
}),
"",
"[derived]",
JSON.stringify(derived),
"",
"[assistant_response]",
responseText,
"",
'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}',
].join("\n");
}
export async function judgeOnboardingWithLlm(
llmClient: LlmClient,
evalCase: EvalCaseDefinition,
responseText: string,
derived: EvalDerivedSnapshot,
): Promise<LlmJudgeResult> {
if (!responseText.trim()) {
return {
skipped: true,
reason: "empty-response",
};
}
if (!llmClient.hasEndpoints()) {
return {
skipped: true,
reason: "no-llm-endpoint",
};
}
const response = await llmClient.chatCompletion(
[
{
role: "system",
content:
"당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
},
{
role: "user",
content: buildJudgePrompt(evalCase, responseText, derived),
},
],
{
jsonMode: true,
maxTokens: getLlmJudgeMaxTokens(),
modelOverride: getLlmJudgeModel(),
temperature: 0,
},
);
if (!response) {
return {
skipped: true,
reason: "judge-call-failed",
};
}
try {
const parsed = llmJudgeSchema.parse(JSON.parse(response.content));
const scores: LlmJudgeScores = {
friendliness: parsed.friendliness,
accuracy: parsed.accuracy,
concision: parsed.concision,
scope: parsed.scope,
overall: round(
(parsed.friendliness +
parsed.accuracy +
parsed.concision +
parsed.scope) /
4,
),
};
return {
skipped: false,
model: getLlmJudgeModel(),
endpoint: response.endpoint,
provider: response.provider,
scores,
rationale: parsed.rationale,
raw: response.content,
};
} catch {
return {
skipped: true,
reason: "invalid-judge-json",
raw: response.content,
};
}
}
+70
View File
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import { judgeOnboardingRuleCase } from "./rule-judge.js";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
function makeDerived(
overrides?: Partial<DerivedState>,
): DerivedState {
return {
phase: { kind: "photo-pending" },
intent: { kind: "status-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
...overrides,
};
}
describe("judgeOnboardingRuleCase", () => {
it("passes when active checks match", () => {
const result = judgeOnboardingRuleCase(
{
intent: "status-question",
responseMode: "llm",
mustIncludeAny: ["방 하트 인증"],
mustNotInclude: ["디스코드 서버"],
},
"이제 방 하트 인증만 남았어요.",
makeDerived(),
);
expect(result.passed).toBe(true);
expect(result.score).toBe(1);
expect(result.validatorStatus).toBe("pass");
});
it("fails intent and mustNotInclude mismatches", () => {
const result = judgeOnboardingRuleCase(
{
intent: "help-how",
mustNotInclude: ["디스코드"],
},
"디스코드 서버에 먼저 들어오시면 돼요.",
makeDerived(),
);
expect(result.passed).toBe(false);
expect(result.checks.find((check) => check.name === "intent")?.passed).toBe(
false,
);
expect(
result.checks.find((check) => check.name === "mustNotInclude")?.passed,
).toBe(false);
});
it("treats rejecting validator results as a failed rule check", () => {
const result = judgeOnboardingRuleCase(
{},
"디스코드 서버에 들어가시면 돼요.",
makeDerived({
intent: { kind: "photo-question" },
}),
);
expect(result.passed).toBe(false);
expect(result.validatorStatus).toBe("reject");
});
});
+121
View File
@@ -0,0 +1,121 @@
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
function normalizeText(value: string): string {
return value.trim().toLowerCase();
}
function round(value: number): number {
return Number(value.toFixed(3));
}
export function judgeOnboardingRuleCase(
expected: EvalCaseExpected,
responseText: string,
derived: DerivedState,
): RuleJudgeResult {
const validation = validateOnboardingResponse(responseText, derived, {
allowShortResponse: derived.allowShortResponse,
});
const checks: RuleJudgeCheck[] = [];
checks.push({
name: "validator",
passed: validation.status !== "reject",
skipped: false,
detail:
validation.reasons.length > 0
? validation.reasons.join(", ")
: validation.status,
});
const expectedIntent = expected.intent?.trim();
if (expectedIntent) {
checks.push({
name: "intent",
passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind),
skipped: false,
detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`,
});
} else {
checks.push({
name: "intent",
passed: true,
skipped: true,
});
}
const expectedResponseMode = expected.responseMode?.trim();
if (expectedResponseMode) {
checks.push({
name: "responseMode",
passed:
normalizeText(expectedResponseMode) ===
normalizeText(derived.responseMode),
skipped: false,
detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`,
});
} else {
checks.push({
name: "responseMode",
passed: true,
skipped: true,
});
}
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
if (mustIncludeAny.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustIncludeAny.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustIncludeAny",
passed: Boolean(matched),
skipped: false,
detail: matched
? `matched=${matched}`
: `expected one of: ${mustIncludeAny.join(", ")}`,
});
} else {
checks.push({
name: "mustIncludeAny",
passed: true,
skipped: true,
});
}
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
if (mustNotInclude.length > 0) {
const normalizedResponse = normalizeText(responseText);
const matched = mustNotInclude.find((candidate) =>
normalizedResponse.includes(normalizeText(candidate)),
);
checks.push({
name: "mustNotInclude",
passed: !matched,
skipped: false,
detail: matched ? `forbidden=${matched}` : undefined,
});
} else {
checks.push({
name: "mustNotInclude",
passed: true,
skipped: true,
});
}
const activeChecks = checks.filter((check) => !check.skipped);
const passedChecks = activeChecks.filter((check) => check.passed).length;
return {
passed: activeChecks.every((check) => check.passed),
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
checks,
validatorStatus: validation.status,
validatorReasons: validation.reasons,
};
}
+56
View File
@@ -0,0 +1,56 @@
import { describe, expect, it, vi } from "vitest";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { publishEvalReportIfPossible } from "./runner.js";
import type { EvalReport } from "./types.js";
const natsMocks = vi.hoisted(() => ({
close: vi.fn().mockResolvedValue(undefined),
connect: vi.fn().mockResolvedValue(undefined),
publish: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@common/nats", () => {
return {
NatsClient: class {
connect = natsMocks.connect;
publish = natsMocks.publish;
close = natsMocks.close;
},
};
});
const report: EvalReport = {
generatedAt: "2026-04-17T00:00:00.000Z",
inputPath: "/tmp/onboarding.jsonl",
setName: "golden",
summary: {
averageLatencyMs: 100,
cases: 1,
emptyResponses: 0,
llmAverageAccuracy: 4,
llmAverageConcision: 4,
llmAverageFriendliness: 5,
llmAverageOverall: 4.25,
llmAverageScope: 4,
llmJudgedCases: 1,
passRate: 1,
passedCases: 1,
ruleAverageScore: 1,
},
cases: [],
};
describe("publishEvalReportIfPossible", () => {
it("publishes the report to the eval report subject", async () => {
const result = await publishEvalReportIfPossible(report);
expect(result).toBe(true);
expect(natsMocks.connect).toHaveBeenCalledTimes(1);
expect(natsMocks.publish).toHaveBeenCalledWith(
AGENT_EVAL_REPORT_SUBJECT,
report,
);
expect(natsMocks.close).toHaveBeenCalledTimes(1);
});
});
+842
View File
@@ -0,0 +1,842 @@
import { existsSync } from "node:fs";
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import { basename, dirname, resolve } from "node:path";
import type { AgentRequest } from "@agent/types";
import { GarnetClient } from "@common/garnet";
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
import { NatsClient } from "@common/nats";
import { getConversationSummaryKey } from "@conv/conversation-store";
import { LlmClient } from "@llm/llm-client";
import type { ProviderName } from "@llm/types";
import { runWithContext } from "@obs/context";
import { getLogger } from "@obs/logger";
import type { RoomMessage } from "@room/types";
import { RoomContextService } from "@room/room-context-service";
import { OnboardingSkill } from "@skills/onboarding/index";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import { OnboardingIntentClassifier } from "@skills/onboarding/state/classifier";
import { deriveState } from "@skills/onboarding/state/derive";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import { noopShadowEvalPublisher } from "./shadow-bus.js";
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl, getOpenRouterEnabled } from "../config.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import { judgeOnboardingRuleCase } from "./judges/rule-judge.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalCaseExpected,
EvalCaseInput,
EvalCaseResult,
EvalClassifiedSnapshot,
EvalDerivedSnapshot,
EvalReport,
EvalReportSummary,
EvalSummaryDelta,
} from "./types.js";
const DEFAULT_REPORTS_DIR = "eval-reports";
const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl";
const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl";
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
class InMemoryGarnetClient extends GarnetClient {
private readonly hashStore = new Map<string, Record<string, string>>();
private readonly stringStore = new Map<string, string>();
getJson<T>(key: string): Promise<T | null> {
const raw = this.stringStore.get(key);
return Promise.resolve(raw ? (JSON.parse(raw) as T) : null);
}
setJson(
key: string,
value: unknown,
_ttlSeconds?: number,
): Promise<void> {
this.stringStore.set(key, JSON.stringify(value));
return Promise.resolve();
}
getString(key: string): Promise<string | null> {
return Promise.resolve(this.stringStore.get(key) ?? null);
}
setString(
key: string,
value: string,
_ttlSeconds?: number,
): Promise<void> {
this.stringStore.set(key, value);
return Promise.resolve();
}
delete(key: string): Promise<number> {
const stringDeleted = this.stringStore.delete(key);
const hashDeleted = this.hashStore.delete(key);
return Promise.resolve(stringDeleted || hashDeleted ? 1 : 0);
}
getHash(key: string): Promise<Record<string, string>> {
return Promise.resolve({ ...(this.hashStore.get(key) ?? {}) });
}
setHash(
key: string,
fields: Record<string, string>,
): Promise<void> {
this.hashStore.set(key, { ...fields });
return Promise.resolve();
}
}
class InMemoryRoomContextService extends RoomContextService {
constructor(private readonly messages: RoomMessage[]) {
super();
}
getRecentMessages(
_chatroomId: string,
limit?: number,
): Promise<RoomMessage[]> {
const effectiveLimit = limit ?? this.messages.length;
return Promise.resolve(this.messages.slice(-effectiveLimit));
}
}
interface RawEvalCaseLine {
expected?: Record<string, unknown>;
id?: unknown;
input?: Record<string, unknown>;
name?: unknown;
notes?: unknown;
}
export interface EvalRunOptions {
enableLlmJudge?: boolean | undefined;
inputPath?: string | undefined;
limit?: number | undefined;
outputPath?: string | undefined;
setName?: string | undefined;
}
function round(value: number): number {
return Number(value.toFixed(3));
}
function average(values: number[]): number | null {
if (values.length === 0) {
return null;
}
return round(values.reduce((sum, value) => sum + value, 0) / values.length);
}
function averageOrZero(values: number[]): number {
return values.length === 0 ? 0 : round(values.reduce((sum, value) => sum + value, 0) / values.length);
}
function firstNonEmpty(
...values: Array<string | null | undefined>
): string | undefined {
for (const value of values) {
if (value?.trim()) {
return value.trim();
}
}
return undefined;
}
function parseEndpointList(
...values: Array<string | null | undefined>
): string[] {
const endpoints: string[] = [];
for (const value of values) {
if (!value?.trim()) {
continue;
}
for (const item of value.split(/[\n,]/u)) {
const trimmed = item.trim();
if (trimmed) {
endpoints.push(trimmed);
}
}
}
return [...new Set(endpoints)];
}
function toProviderEndpoints(
urls: string[],
provider?: ProviderName,
): Array<string | { url: string; provider?: ProviderName | undefined }> {
if (!provider) {
return urls;
}
return urls.map((url) => ({ url, provider }));
}
function createEvalLlmClient(): LlmClient {
const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]);
const llamaEndpoints = parseEndpointList(
process.env["AGENT_LLM_API_URLS"],
process.env["LLAMA_SERVER_API_URL"],
);
const openRouterEndpoints = getOpenRouterEnabled()
? parseEndpointList(process.env["OPENROUTER_API_URL"], getOpenRouterApiUrl())
: [];
const llmApiKey = firstNonEmpty(
process.env["AGENT_LLM_API_KEY"],
process.env["LLAMA_SERVER_API_KEY"],
);
const providerApiKeys = Object.fromEntries(
[
["cliproxy", firstNonEmpty(process.env["CLI_PROXY_API_KEY"])],
["llama", llmApiKey],
["openrouter", firstNonEmpty(process.env["OPENROUTER_API_KEY"])],
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
) as Partial<Record<ProviderName, string>>;
return new LlmClient({
endpoints: [
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
...toProviderEndpoints(llamaEndpoints, "llama"),
],
apiKey: llmApiKey,
model: firstNonEmpty(process.env["AGENT_LLM_MODEL"]),
providerApiKeys,
});
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
function getStringField(record: Record<string, unknown>, key: string): string | undefined {
const value = record[key];
return typeof value === "string" ? value : undefined;
}
function getStringArrayField(
record: Record<string, unknown>,
key: string,
): string[] | undefined {
const value = record[key];
if (!Array.isArray(value)) {
return undefined;
}
const strings = value.filter((item): item is string => typeof item === "string");
return strings.length > 0 ? strings : [];
}
function toAgentRequest(raw: Record<string, unknown>): AgentRequest {
const requestId = getStringField(raw, "requestId");
const skill = getStringField(raw, "skill");
const chatroomId = getStringField(raw, "chatroomId");
const userId = getStringField(raw, "userId");
const nickname = getStringField(raw, "nickname");
const trigger = getStringField(raw, "trigger");
if (!requestId || !skill || !chatroomId || !userId || !nickname || !trigger) {
throw new Error("input.request is missing required AgentRequest fields");
}
const request: AgentRequest = {
requestId,
skill,
chatroomId,
userId,
nickname,
trigger,
};
const userMessage = getStringField(raw, "userMessage");
if (userMessage !== undefined) {
request.userMessage = userMessage;
}
const context = raw["context"];
if (isRecord(context)) {
request.context = context;
}
return request;
}
function parseConversationHistory(raw: unknown): EvalCaseInput["conversationHistory"] {
if (!Array.isArray(raw)) {
return undefined;
}
return raw
.filter((item): item is Record<string, unknown> => isRecord(item))
.map((item) => {
const role = item["role"];
const content = getStringField(item, "content") ?? "";
const timestamp = getStringField(item, "timestamp") ?? new Date(0).toISOString();
if (role !== "user" && role !== "assistant") {
throw new Error("conversationHistory contains an invalid role");
}
return {
role,
content,
timestamp,
};
});
}
function parseRoomContext(raw: unknown): EvalCaseInput["roomContext"] {
if (!Array.isArray(raw)) {
return undefined;
}
return raw
.filter((item): item is Record<string, unknown> => isRecord(item))
.map((item) => ({
senderId: getStringField(item, "senderId") ?? "",
nickname: getStringField(item, "nickname") ?? "알 수 없음",
message: getStringField(item, "message") ?? "",
timestamp: getStringField(item, "timestamp") ?? new Date(0).toISOString(),
}));
}
function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpected {
if (!raw) {
return {};
}
const expected: EvalCaseExpected = {};
const intent = getStringField(raw, "intent");
const responseMode = getStringField(raw, "responseMode");
const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny");
const mustNotInclude = getStringArrayField(raw, "mustNotInclude");
if (intent) {
expected.intent = intent;
}
if (responseMode) {
expected.responseMode = responseMode;
}
if (mustIncludeAny) {
expected.mustIncludeAny = mustIncludeAny;
}
if (mustNotInclude) {
expected.mustNotInclude = mustNotInclude;
}
return expected;
}
function parseEvalCaseLine(
raw: RawEvalCaseLine,
lineNumber: number,
sourcePath: string,
): EvalCaseDefinition {
if (!isRecord(raw.input)) {
throw new Error("input is required");
}
const requestRaw = raw.input["request"];
if (!isRecord(requestRaw)) {
throw new Error("input.request is required");
}
const input: EvalCaseInput = {
request: toAgentRequest(requestRaw),
};
const conversationHistory = parseConversationHistory(raw.input["conversationHistory"]);
if (conversationHistory) {
input.conversationHistory = conversationHistory;
}
const roomContext = parseRoomContext(raw.input["roomContext"]);
if (roomContext) {
input.roomContext = roomContext;
}
if (raw.input["summary"] !== undefined) {
const summary = getStringField(raw.input, "summary");
if (summary !== undefined) {
input.summary = summary;
}
}
if (raw.input["session"] === null || isRecord(raw.input["session"])) {
input.session = (raw.input["session"] ?? null) as OnboardingSession | null;
}
if (raw.input["blackboard"] === null || isRecord(raw.input["blackboard"])) {
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
}
return {
id:
typeof raw.id === "string" && raw.id.trim()
? raw.id.trim()
: `${basename(sourcePath)}:${lineNumber}`,
lineNumber,
name: typeof raw.name === "string" ? raw.name : undefined,
notes: typeof raw.notes === "string" ? raw.notes : undefined,
input,
expected: parseExpected(isRecord(raw.expected) ? raw.expected : undefined),
};
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
return {
allowShortResponse: derived.allowShortResponse,
intent: derived.intent.kind,
phase: derived.phase.kind,
responseMode: derived.responseMode,
shouldRespond: derived.shouldRespond,
suggestedMaxTokens: derived.suggestedMaxTokens,
tier: derived.tier,
};
}
function toClassifiedSnapshot(classified: ClassifiedIntent): EvalClassifiedSnapshot {
return {
confidence: round(classified.confidence),
intent: classified.intent.kind,
source: classified.source,
};
}
function toRoomMessages(
input: EvalCaseInput["roomContext"],
): RoomMessage[] {
return (
input?.map((message) => ({
senderId: message.senderId,
nickname: message.nickname,
message: message.message,
timestamp: new Date(message.timestamp),
})) ?? []
);
}
async function hydrateEvalState(
garnetClient: InMemoryGarnetClient,
input: EvalCaseInput,
): Promise<void> {
const {
request: { chatroomId, userId },
} = input;
if (input.session !== undefined) {
await garnetClient.setJson(
`${getOnboardingSessionKeyPrefix()}:onboarding:${chatroomId}:${userId}`,
input.session,
);
}
if (input.blackboard !== undefined) {
await garnetClient.setJson(
`lolympus-agent:bb:${chatroomId}:${userId}`,
input.blackboard,
);
}
if (input.summary?.trim()) {
await garnetClient.setString(
getConversationSummaryKey(chatroomId, userId),
input.summary,
);
}
}
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
const llmResults = results.flatMap((result) =>
result.llmJudge.skipped ? [] : [result.llmJudge],
);
return {
averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)),
cases: results.length,
emptyResponses: results.filter((result) => !result.response.text.trim()).length,
llmAverageAccuracy: average(
llmResults.map((result) => result.scores.accuracy),
),
llmAverageConcision: average(
llmResults.map((result) => result.scores.concision),
),
llmAverageFriendliness: average(
llmResults.map((result) => result.scores.friendliness),
),
llmAverageOverall: average(
llmResults.map((result) => result.scores.overall),
),
llmAverageScope: average(
llmResults.map((result) => result.scores.scope),
),
llmJudgedCases: llmResults.length,
passRate:
results.length === 0
? 0
: round(
results.filter((result) => result.ruleJudge.passed).length /
results.length,
),
passedCases: results.filter((result) => result.ruleJudge.passed).length,
ruleAverageScore: averageOrZero(
results.map((result) => result.ruleJudge.score),
),
};
}
function createBaselineDelta(
current: EvalReportSummary,
baseline: EvalReportSummary,
): EvalSummaryDelta {
const subtract = (
left: number | null,
right: number | null,
): number | null => {
if (left === null || right === null) {
return null;
}
return round(left - right);
};
return {
averageLatencyMs: subtract(current.averageLatencyMs, baseline.averageLatencyMs),
llmAverageOverall: subtract(
current.llmAverageOverall,
baseline.llmAverageOverall,
),
passRate: subtract(current.passRate, baseline.passRate),
ruleAverageScore: subtract(current.ruleAverageScore, baseline.ruleAverageScore),
};
}
function buildDefaultReportPath(timestamp: string): string {
const filename = `${timestamp.replace(/[:.]/gu, "-")}.json`;
return resolve(process.cwd(), DEFAULT_REPORTS_DIR, filename);
}
async function writeReport(
report: EvalReport,
outputPath: string,
): Promise<string> {
const resolved = resolve(process.cwd(), outputPath);
await mkdir(dirname(resolved), { recursive: true });
await writeFile(resolved, `${JSON.stringify(report, null, 2)}\n`, "utf8");
return resolved;
}
async function loadBaselineFile(): Promise<EvalBaselineFile | null> {
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
if (!existsSync(baselinePath)) {
return null;
}
const raw = await readFile(baselinePath, "utf8");
return JSON.parse(raw) as EvalBaselineFile;
}
export function resolveEvalSetPath(setNameOrPath = "golden"): string {
if (setNameOrPath === "golden") {
return resolve(process.cwd(), GOLDEN_SET_PATH);
}
if (setNameOrPath === "capture") {
return resolve(process.cwd(), CAPTURE_SET_PATH);
}
if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) {
return resolve(process.cwd(), setNameOrPath);
}
return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`);
}
export async function loadEvalCases(
inputPath: string,
options?: { limit?: number | undefined },
): Promise<EvalCaseDefinition[]> {
const resolvedPath = resolve(process.cwd(), inputPath);
if (!existsSync(resolvedPath)) {
throw new Error(`Eval set not found: ${resolvedPath}`);
}
const raw = await readFile(resolvedPath, "utf8");
const lines = raw.split(/\r?\n/u).filter((line) => line.trim());
const cases = lines.map((line, index) =>
parseEvalCaseLine(
JSON.parse(line) as RawEvalCaseLine,
index + 1,
resolvedPath,
),
);
return options?.limit ? cases.slice(0, options.limit) : cases;
}
async function evaluateCase(
evalCase: EvalCaseDefinition,
llmClient: LlmClient,
enableLlmJudge: boolean,
): Promise<EvalCaseResult> {
const input = evalCase.input;
const inMemoryGarnet = new InMemoryGarnetClient();
await hydrateEvalState(inMemoryGarnet, input);
const roomContextService =
input.roomContext && input.roomContext.length > 0
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
: undefined;
const skill = new OnboardingSkill(
inMemoryGarnet,
llmClient,
roomContextService,
noopShadowEvalPublisher,
);
const classifier = new OnboardingIntentClassifier(llmClient);
return runWithContext(
{
chatroomId: input.request.chatroomId,
deadlineAt: new Date(Date.now() + 4_500).toISOString(),
requestId: input.request.requestId,
skill: input.request.skill,
startedAt: new Date().toISOString(),
trigger: input.request.trigger,
userId: input.request.userId,
},
async () => {
const classified = await classifier.classify(
input.request.userMessage,
input.request.trigger,
);
const derived = deriveState(
input.session ?? null,
input.request.trigger,
classified.intent,
);
const startedAt = Date.now();
try {
const response = await skill.handle(
input.request,
input.conversationHistory ?? [],
);
const latencyMs = Date.now() - startedAt;
const derivedSnapshot = toDerivedSnapshot(derived);
const ruleJudge = judgeOnboardingRuleCase(
evalCase.expected,
response.text,
derived,
);
const llmJudge = enableLlmJudge
? await judgeOnboardingWithLlm(
llmClient,
evalCase,
response.text,
derivedSnapshot,
)
: {
skipped: true as const,
reason: "disabled",
};
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: derivedSnapshot,
request: input.request,
response,
ruleJudge,
llmJudge,
};
} catch (error) {
const latencyMs = Date.now() - startedAt;
const errorMessage =
error instanceof Error ? error.message : String(error);
return {
id: evalCase.id,
lineNumber: evalCase.lineNumber,
name: evalCase.name,
latencyMs,
classified: toClassifiedSnapshot(classified),
derived: toDerivedSnapshot(derived),
request: input.request,
response: {
requestId: input.request.requestId,
text: "",
confidence: 0,
source: "template",
metadata: {
error: errorMessage,
},
},
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
llmJudge: {
skipped: true,
reason: "skill-error",
},
error: errorMessage,
};
}
},
);
}
export async function runOnboardingEval(
options?: EvalRunOptions,
): Promise<{ report: EvalReport; reportPath: string }> {
const logger = getLogger();
const setName = options?.setName ?? "golden";
const inputPath = options?.inputPath
? resolve(process.cwd(), options.inputPath)
: resolveEvalSetPath(setName);
const evalCases = await loadEvalCases(inputPath, {
limit: options?.limit,
});
const llmClient = createEvalLlmClient();
const enableLlmJudge = options?.enableLlmJudge ?? true;
const caseResults: EvalCaseResult[] = [];
for (const evalCase of evalCases) {
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge));
}
const summary = createSummary(caseResults);
const baseline = await loadBaselineFile();
const report: EvalReport = {
generatedAt: new Date().toISOString(),
inputPath,
setName,
summary,
cases: caseResults,
};
if (baseline) {
report.baselineDelta = createBaselineDelta(summary, baseline.summary);
}
const reportPath = await writeReport(
report,
options?.outputPath ?? buildDefaultReportPath(report.generatedAt),
);
logger.info(
{
cases: summary.cases,
passRate: summary.passRate,
reportPath,
},
"[Eval] run completed",
);
return { report, reportPath };
}
export async function findLatestEvalReportPath(): Promise<string | null> {
const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR);
if (!existsSync(reportsDir)) {
return null;
}
const entries = await readdir(reportsDir, { withFileTypes: true });
const candidates = entries
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
.filter((name) => name.endsWith(".json") && name !== "baseline.json")
.sort((left, right) => right.localeCompare(left));
if (candidates.length === 0) {
return null;
}
const latest = candidates[0];
if (!latest) {
return null;
}
return resolve(reportsDir, latest);
}
export async function updateEvalBaseline(
reportPath?: string,
): Promise<{
baseline: EvalBaselineFile;
baselinePath: string;
storedInRedis: boolean;
}> {
const sourcePath =
reportPath?.trim() ? resolve(process.cwd(), reportPath) : await findLatestEvalReportPath();
if (!sourcePath) {
throw new Error("No eval report found to baseline");
}
const raw = await readFile(sourcePath, "utf8");
const report = JSON.parse(raw) as EvalReport;
const baseline: EvalBaselineFile = {
reportPath: sourcePath,
setName: report.setName,
summary: report.summary,
updatedAt: new Date().toISOString(),
};
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
await mkdir(dirname(baselinePath), { recursive: true });
await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
const storedInRedis = await persistBaselineInRedis(baseline);
return { baseline, baselinePath, storedInRedis };
}
async function persistBaselineInRedis(
baseline: EvalBaselineFile,
): Promise<boolean> {
try {
const { GarnetClient } = await import("@common/garnet");
const garnetClient = new GarnetClient();
await garnetClient.connect();
try {
await garnetClient.setString(
EVAL_BASELINE_KEY,
JSON.stringify(baseline),
);
return true;
} finally {
await garnetClient.close();
}
} catch {
return false;
}
}
export async function publishEvalReportIfPossible(
report: EvalReport,
): Promise<boolean> {
try {
const natsClient = new NatsClient();
await natsClient.connect();
try {
await natsClient.publish(AGENT_EVAL_REPORT_SUBJECT, report);
return true;
} finally {
await natsClient.close();
}
} catch {
return false;
}
}
+93
View File
@@ -0,0 +1,93 @@
import { describe, expect, it, vi } from "vitest";
import type { AgentRequest, AgentResponse } from "@agent/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
import {
ShadowEvalBus,
shouldSampleShadowEval,
} from "./shadow-bus.js";
const request: AgentRequest = {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "프리리/95/여/D",
trigger: "user-message",
userMessage: "이제 뭐 남았지",
};
const response: AgentResponse = {
requestId: "req-1",
text: "이제 방 하트 인증만 남았어요.",
confidence: 0.9,
source: "llm",
};
const classified: ClassifiedIntent = {
intent: { kind: "status-question" },
confidence: 0.98,
source: "regex",
};
const derived: DerivedState = {
phase: { kind: "photo-pending" },
intent: { kind: "status-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
};
describe("shouldSampleShadowEval", () => {
it("samples when the random value is under the tier rate", () => {
expect(shouldSampleShadowEval(0, 0)).toBe(true);
});
it("does not sample when the random value exceeds the tier rate", () => {
expect(shouldSampleShadowEval(0, 1)).toBe(false);
});
});
describe("ShadowEvalBus", () => {
it("publishes a shadow-eval stream entry when sampled", () => {
const appendStream = vi.fn().mockResolvedValue("1-0");
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
new ShadowEvalBus({ appendStream }).publish({
classified,
createdAt: "2026-04-16T00:00:00.000Z",
derived,
request,
response,
});
expect(appendStream).toHaveBeenCalledTimes(1);
expect(appendStream.mock.calls[0]?.[0]).toBe("lolympus-agent:background");
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
type: "shadow-eval",
requestId: "req-1",
tier: "0",
});
randomSpy.mockRestore();
});
it("skips publishing for empty responses", () => {
const appendStream = vi.fn().mockResolvedValue("1-0");
new ShadowEvalBus({ appendStream }).publish({
classified,
createdAt: "2026-04-16T00:00:00.000Z",
derived,
request,
response: {
...response,
text: "",
},
});
expect(appendStream).not.toHaveBeenCalled();
});
});
+90
View File
@@ -0,0 +1,90 @@
import type { AgentRequest, AgentResponse } from "@agent/types";
import type { GarnetClient } from "@common/garnet";
import { getRequestLogger } from "@obs/context";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import {
getBackgroundStreamMaxLen,
getShadowEvalSamplingRate,
} from "../config.js";
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
export interface ShadowEvalEnvelope {
classified: ClassifiedIntent;
createdAt: string;
derived: DerivedState;
latencyMs?: number | undefined;
request: AgentRequest;
response: AgentResponse;
}
export interface ShadowEvalPublisher {
publish(envelope: ShadowEvalEnvelope): void;
}
export const noopShadowEvalPublisher: ShadowEvalPublisher = {
publish(): void {},
};
export function shouldSampleShadowEval(
tier: 0 | 1 | 2,
randomValue = Math.random(),
): boolean {
return randomValue < getShadowEvalSamplingRate(tier);
}
export class ShadowEvalBus implements ShadowEvalPublisher {
constructor(
private readonly garnetClient: Pick<GarnetClient, "appendStream">,
) {}
publish(envelope: ShadowEvalEnvelope): void {
if (!envelope.response.text.trim()) {
return;
}
if (!shouldSampleShadowEval(envelope.derived.tier)) {
return;
}
const logger = getRequestLogger();
void this.garnetClient
.appendStream(
BACKGROUND_STREAM_KEY,
{
type: "shadow-eval",
requestId: envelope.request.requestId,
chatroomId: envelope.request.chatroomId,
userId: envelope.request.userId,
tier: String(envelope.derived.tier),
latencyMs:
envelope.latencyMs !== undefined
? String(envelope.latencyMs)
: "",
input: JSON.stringify({
request: envelope.request,
}),
output: JSON.stringify(envelope.response),
derived: JSON.stringify(envelope.derived),
classified: JSON.stringify(envelope.classified),
createdAt: envelope.createdAt,
},
{
maxLen: getBackgroundStreamMaxLen(),
},
)
.catch((error) => {
logger.warn(
{
err: error,
requestId: envelope.request.requestId,
},
"[ShadowEvalBus] Failed to publish shadow eval job",
);
});
}
}
+53
View File
@@ -0,0 +1,53 @@
import { describe, expect, it } from "vitest";
import {
calculateEvalScoreDropPct,
calculateLlmErrorRate,
calculatePercentile,
} from "./shadow-worker.js";
describe("calculatePercentile", () => {
it("returns null for empty input", () => {
expect(calculatePercentile([], 0.95)).toBeNull();
});
it("calculates the p95 from sorted values", () => {
expect(calculatePercentile([100, 200, 300, 400, 500], 0.95)).toBe(500);
});
});
describe("calculateEvalScoreDropPct", () => {
it("returns null without a valid baseline", () => {
expect(calculateEvalScoreDropPct(4.2, null)).toBeNull();
expect(calculateEvalScoreDropPct(4.2, 0)).toBeNull();
});
it("computes percentage drop from baseline", () => {
expect(calculateEvalScoreDropPct(3.6, 4.0)).toBeCloseTo(10);
});
});
describe("calculateLlmErrorRate", () => {
it("returns null when no observations are present", () => {
expect(calculateLlmErrorRate([])).toBeNull();
});
it("computes error ratio across observations", () => {
expect(
calculateLlmErrorRate([
{
llmJudgeErrored: true,
llmOverall: null,
recordedAt: 1,
responseLatencyMs: 100,
},
{
llmJudgeErrored: false,
llmOverall: 4.2,
recordedAt: 2,
responseLatencyMs: 200,
},
]),
).toBe(0.5);
});
});
+741
View File
@@ -0,0 +1,741 @@
import { existsSync } from "node:fs";
import { readFile } from "node:fs/promises";
import { resolve } from "node:path";
import type { AgentRequest, AgentResponse } from "@agent/types";
import type {
GarnetClient,
StreamEntry,
StreamPendingEntry,
} from "@common/garnet";
import { summarizeConversation } from "@conv/summarizer";
import type { LlmClient } from "@llm/llm-client";
import { getLogger } from "@obs/logger";
import type { AlertSender } from "@obs/alert";
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ConversationStore } from "@conv/conversation-store";
import {
getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs,
getEvalStreamMaxLen,
} from "../config.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalDerivedSnapshot,
} from "./types.js";
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
const BACKGROUND_GROUP = "bg-worker";
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
const ALERT_WINDOW_MS = 5 * 60_000;
const ALERT_WINDOW_LABEL = "5m";
const ALERT_TICK_MS = 60_000;
const MAX_DELIVERIES = 3;
const MIN_ERROR_RATE_OBSERVATIONS = 5;
const MIN_LATENCY_OBSERVATIONS = 5;
const MIN_SCORE_OBSERVATIONS = 3;
const PENDING_IDLE_MS = 1_000;
const READ_BLOCK_MS = 5_000;
const READ_COUNT = 10;
const PENDING_BATCH_SIZE = 20;
const STOP_TIMEOUT_MS = 10_000;
type BackgroundJobType = "shadow-eval" | "summarize";
interface ShadowEvalJob {
classified: ClassifiedIntent;
createdAt: string;
derived: DerivedState;
latencyMs?: number | undefined;
request: AgentRequest;
response: AgentResponse;
}
interface EvalObservation {
llmJudgeErrored: boolean;
llmOverall: number | null;
recordedAt: number;
responseLatencyMs: number | null;
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
function promiseWithTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
): Promise<T> {
return Promise.race([
promise,
new Promise<T>((_, reject) => {
setTimeout(() => {
reject(new Error(`Timed out after ${timeoutMs}ms`));
}, timeoutMs);
}),
]);
}
function getJobType(message: StreamEntry): string {
return message.message["type"] ?? "unknown";
}
function parseOptionalNumberField(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : undefined;
}
function parseJsonField<T>(
value: string | undefined,
fieldName: string,
): T {
if (!value?.trim()) {
throw new Error(`Missing ${fieldName} field`);
}
return JSON.parse(value) as T;
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
return {
allowShortResponse: derived.allowShortResponse,
intent: derived.intent.kind,
phase: derived.phase.kind,
responseMode: derived.responseMode,
shouldRespond: derived.shouldRespond,
suggestedMaxTokens: derived.suggestedMaxTokens,
tier: derived.tier,
};
}
function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition {
return {
id: `shadow:${messageId}`,
lineNumber: 0,
input: {
request: job.request,
},
expected: {},
};
}
function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
const input = parseJsonField<{ request: AgentRequest }>(
message.message["input"],
"input",
);
return {
classified: parseJsonField<ClassifiedIntent>(
message.message["classified"],
"classified",
),
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
derived: parseJsonField<DerivedState>(
message.message["derived"],
"derived",
),
latencyMs: parseOptionalNumberField(message.message["latencyMs"]),
request: input.request,
response: parseJsonField<AgentResponse>(
message.message["output"],
"output",
),
};
}
export function calculatePercentile(
values: number[],
percentile: number,
): number | null {
if (values.length === 0) {
return null;
}
const sorted = [...values].sort((left, right) => left - right);
const index = Math.min(
sorted.length - 1,
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
);
const selected = sorted[index];
return selected === undefined ? null : selected;
}
export function calculateEvalScoreDropPct(
currentAverage: number | null,
baselineAverage: number | null,
): number | null {
if (
currentAverage === null ||
baselineAverage === null ||
baselineAverage <= 0
) {
return null;
}
return ((baselineAverage - currentAverage) / baselineAverage) * 100;
}
export function calculateLlmErrorRate(
observations: EvalObservation[],
): number | null {
if (observations.length === 0) {
return null;
}
const errorCount = observations.filter(
(observation) => observation.llmJudgeErrored,
).length;
return errorCount / observations.length;
}
export class ShadowWorker {
private lastAggregateAt = 0;
private loopPromise: Promise<void> | null = null;
private readonly logger = getLogger();
private readonly recentEvalObservations: EvalObservation[] = [];
private stopRequested = false;
constructor(
private readonly garnetClient: GarnetClient,
private readonly llmClient: LlmClient,
private readonly conversationStore: ConversationStore,
private readonly alertSender?: AlertSender,
private readonly consumerName = `bg-worker-${process.pid}`,
) {}
async start(): Promise<void> {
await this.ensureConsumerGroup();
if (this.loopPromise) {
return;
}
this.stopRequested = false;
this.loopPromise = this.runLoop();
}
async stop(): Promise<void> {
this.stopRequested = true;
if (!this.loopPromise) {
return;
}
try {
await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS);
} catch (error) {
this.logger.warn(
{ err: error },
"[ShadowWorker] Stop timed out while waiting for the loop to finish",
);
} finally {
this.loopPromise = null;
}
}
private async ensureConsumerGroup(): Promise<void> {
try {
await this.garnetClient.createConsumerGroup(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
);
} catch (error) {
if (!String(error).includes("BUSYGROUP")) {
throw error;
}
}
}
private async runLoop(): Promise<void> {
while (!this.stopRequested) {
try {
await this.runPeriodicChecksIfNeeded();
const recoveredPending = await this.recoverPendingOnce();
if (recoveredPending) {
continue;
}
const streams = await this.garnetClient.readGroup(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
this.consumerName,
">",
{
blockMs: READ_BLOCK_MS,
count: READ_COUNT,
},
);
if (!streams) {
continue;
}
for (const stream of streams) {
for (const message of stream.messages) {
if (this.stopRequested) {
return;
}
await this.processMessage(
message,
message.deliveriesCounter ?? 1,
);
}
}
} catch (error) {
this.logger.error(
{ err: error },
"[ShadowWorker] Loop iteration failed",
);
await delay(1_000);
}
}
}
private async recoverPendingOnce(): Promise<boolean> {
const pending = await this.garnetClient.getPendingStreamRange(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
PENDING_BATCH_SIZE,
);
const recoverable = pending.filter(
(entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS,
);
if (recoverable.length === 0) {
return false;
}
for (const entry of recoverable) {
if (this.stopRequested) {
break;
}
await this.recoverPendingEntry(entry);
}
return true;
}
private async recoverPendingEntry(
entry: StreamPendingEntry,
): Promise<void> {
const nextDeliveryCount = entry.deliveriesCounter + 1;
const claimed = await this.garnetClient.claimPendingStream(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
this.consumerName,
PENDING_IDLE_MS,
entry.id,
{
retryCount: nextDeliveryCount,
},
);
const message = claimed[0];
if (!message) {
return;
}
if (nextDeliveryCount > MAX_DELIVERIES) {
await this.handleMaxDeliveries(
message,
nextDeliveryCount,
"recovered entry exceeded max deliveries",
);
return;
}
await this.processMessage(message, nextDeliveryCount);
}
private async processMessage(
message: StreamEntry,
deliveryCount: number,
): Promise<void> {
try {
switch (getJobType(message) as BackgroundJobType) {
case "shadow-eval":
await this.processShadowEval(message);
await this.acknowledge(message.id);
return;
case "summarize":
await this.processSummarize(message);
await this.acknowledge(message.id);
return;
default:
this.logger.warn(
{
messageId: message.id,
type: getJobType(message),
},
"[ShadowWorker] Unknown background job type; acknowledging",
);
await this.acknowledge(message.id);
}
} catch (error) {
this.logger.warn(
{
err: error,
deliveryCount,
messageId: message.id,
type: getJobType(message),
},
"[ShadowWorker] Job processing failed",
);
if (deliveryCount >= MAX_DELIVERIES) {
await this.handleMaxDeliveries(
message,
deliveryCount,
error instanceof Error ? error.message : String(error),
);
}
}
}
private async processShadowEval(message: StreamEntry): Promise<void> {
const job = parseShadowEvalJob(message);
const validation = validateOnboardingResponse(
job.response.text,
job.derived,
{
allowShortResponse: job.derived.allowShortResponse,
},
);
const llmJudge = await judgeOnboardingWithLlm(
this.llmClient,
toEvalCase(job, message.id),
job.response.text,
toDerivedSnapshot(job.derived),
);
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
const llmJudgeErrored =
llmJudge.skipped &&
llmJudge.reason !== "empty-response" &&
llmJudge.reason !== "disabled";
this.recordEvalObservation({
llmJudgeErrored,
llmOverall,
recordedAt: Date.now(),
responseLatencyMs: job.latencyMs ?? null,
});
await this.garnetClient.appendStream(
EVAL_RESULTS_STREAM_KEY,
{
type: "shadow-eval-result",
requestId: job.request.requestId,
chatroomId: job.request.chatroomId,
userId: job.request.userId,
tier: String(job.derived.tier),
intent: job.derived.intent.kind,
responseMode: job.derived.responseMode,
validatorStatus: validation.status,
latencyMs:
job.latencyMs !== undefined ? String(job.latencyMs) : "",
llmOverall:
llmOverall === null ? "" : String(llmOverall),
input: JSON.stringify({
request: job.request,
}),
output: JSON.stringify(job.response),
derived: JSON.stringify(job.derived),
classified: JSON.stringify(job.classified),
validation: JSON.stringify(validation),
llmJudge: JSON.stringify(llmJudge),
evaluatedAt: new Date().toISOString(),
},
{
maxLen: getEvalStreamMaxLen(),
},
);
}
private async processSummarize(message: StreamEntry): Promise<void> {
const chatroomId = message.message["chatroomId"];
const userId = message.message["userId"];
if (!chatroomId?.trim() || !userId?.trim()) {
throw new Error("Missing summarize job identifiers");
}
const [history, existingSummary] = await Promise.all([
this.conversationStore.getHistory(chatroomId, userId),
this.conversationStore.getSummary(chatroomId, userId),
]);
if (history.length === 0) {
return;
}
const summary = await summarizeConversation(
this.llmClient,
history,
existingSummary ?? undefined,
);
await this.conversationStore.setSummary(chatroomId, userId, summary);
}
private async handleMaxDeliveries(
message: StreamEntry,
deliveryCount: number,
lastError: string,
): Promise<void> {
const type = getJobType(message);
if (type === "summarize") {
this.logger.warn(
{
deliveryCount,
lastError,
messageId: message.id,
},
"[ShadowWorker] Dropping summarize job after max retries",
);
await this.acknowledge(message.id);
return;
}
await this.garnetClient.appendStream(
BACKGROUND_DLQ_STREAM_KEY,
{
type,
payload: JSON.stringify(message.message),
retryCount: String(deliveryCount),
lastError: lastError.slice(0, 500),
originalId: message.id,
},
{
maxLen: 1_000,
},
);
await this.acknowledge(message.id);
}
private acknowledge(id: string): Promise<number> {
return this.garnetClient.acknowledgeStream(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
id,
);
}
private recordEvalObservation(observation: EvalObservation): void {
this.recentEvalObservations.push(observation);
this.pruneOldObservations(Date.now());
}
private pruneOldObservations(now: number): void {
const cutoff = now - ALERT_WINDOW_MS;
let removeCount = 0;
for (const observation of this.recentEvalObservations) {
if (observation.recordedAt >= cutoff) {
break;
}
removeCount += 1;
}
if (removeCount > 0) {
this.recentEvalObservations.splice(0, removeCount);
}
}
private async runPeriodicChecksIfNeeded(): Promise<void> {
if (!this.alertSender) {
return;
}
const now = Date.now();
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
return;
}
this.lastAggregateAt = now;
this.pruneOldObservations(now);
await this.checkLatencyAlert(now);
await this.checkEvalScoreDropAlert(now);
await this.checkLlmErrorRateAlert(now);
await this.checkOpenCircuitAlerts(now);
}
private async checkLatencyAlert(now: number): Promise<void> {
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
);
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
return;
}
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
const threshold = getAlertThresholdP95LatencyMs();
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: p95LatencyMs,
window: ALERT_WINDOW_LABEL,
});
}
private async checkEvalScoreDropAlert(now: number): Promise<void> {
if (!this.alertSender) {
return;
}
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
observation.llmOverall === null ? [] : [observation.llmOverall],
);
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
return;
}
const currentAverage =
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
const baselineAverage = await this.loadBaselineOverallScore();
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
const threshold = getAlertThresholdEvalScoreDropPct();
if (dropPct === null || dropPct <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "eval-score-drop",
metric: "shadow_eval.llm_overall_drop_pct",
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(dropPct.toFixed(3)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkLlmErrorRateAlert(now: number): Promise<void> {
if (
!this.alertSender ||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
) {
return;
}
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
const threshold = getAlertThresholdLlmErrorRate();
if (errorRate === null || errorRate <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "llm-error-rate",
metric: "shadow_eval.llm_error_rate",
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(errorRate.toFixed(4)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkOpenCircuitAlerts(now: number): Promise<void> {
if (!this.alertSender) {
return;
}
const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*");
for (const key of circuitKeys) {
const record = await this.garnetClient.getHash(key);
if (record["state"] !== "open") {
continue;
}
const endpointHash = key.split(":").at(-1) ?? "unknown";
const failureCount = Number(record["failureCount"]);
await this.alertSender.sendAlert(
{
kind: "endpoint-down",
metric: "llm.circuit.open",
severity: "critical",
suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.",
threshold: 0,
timestamp: new Date(now).toISOString(),
value: Number.isFinite(failureCount) ? failureCount : 1,
window: ALERT_WINDOW_LABEL,
},
{
dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`,
},
);
}
}
private async loadBaselineOverallScore(): Promise<number | null> {
const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY);
const redisParsed = parseBaselineOverallFromRaw(redisBaseline);
if (redisParsed !== null) {
return redisParsed;
}
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
if (!existsSync(filePath)) {
return null;
}
try {
const fileRaw = await readFile(filePath, "utf8");
return parseBaselineOverallFromRaw(fileRaw);
} catch (error) {
this.logger.warn(
{ err: error, filePath },
"[ShadowWorker] Failed to read baseline file",
);
return null;
}
}
}
function parseBaselineOverallFromRaw(raw: string | null): number | null {
if (!raw?.trim()) {
return null;
}
try {
const parsed = JSON.parse(raw) as Partial<EvalBaselineFile>;
const value = parsed.summary?.llmAverageOverall;
return typeof value === "number" ? value : null;
} catch {
return null;
}
}
+151
View File
@@ -0,0 +1,151 @@
import type {
AgentRequest,
AgentResponse,
ConversationMessage,
} from "@agent/types";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
export interface EvalRoomContextMessage {
senderId: string;
nickname: string;
message: string;
timestamp: string;
}
export interface EvalCaseInput {
request: AgentRequest;
conversationHistory?: ConversationMessage[] | undefined;
roomContext?: EvalRoomContextMessage[] | undefined;
session?: OnboardingSession | null | undefined;
blackboard?: Blackboard | null | undefined;
summary?: string | undefined;
}
export interface EvalCaseExpected {
intent?: string | undefined;
mustIncludeAny?: string[] | undefined;
mustNotInclude?: string[] | undefined;
responseMode?: string | undefined;
}
export interface EvalCaseDefinition {
id: string;
lineNumber: number;
name?: string | undefined;
notes?: string | undefined;
input: EvalCaseInput;
expected: EvalCaseExpected;
}
export interface RuleJudgeCheck {
name:
| "validator"
| "intent"
| "responseMode"
| "mustIncludeAny"
| "mustNotInclude";
passed: boolean;
skipped: boolean;
detail?: string | undefined;
}
export interface RuleJudgeResult {
passed: boolean;
score: number;
checks: RuleJudgeCheck[];
validatorStatus: "pass" | "soft-warn" | "reject";
validatorReasons: string[];
}
export interface LlmJudgeScores {
friendliness: number;
accuracy: number;
concision: number;
scope: number;
overall: number;
}
export type LlmJudgeResult =
| {
skipped: true;
reason: string;
raw?: string | undefined;
}
| {
skipped: false;
model: string;
endpoint: string;
provider: string;
scores: LlmJudgeScores;
rationale: string;
raw: string;
};
export interface EvalDerivedSnapshot {
allowShortResponse: boolean;
intent: string;
phase: string;
responseMode: DerivedState["responseMode"];
shouldRespond: boolean;
suggestedMaxTokens: number;
tier: DerivedState["tier"];
}
export interface EvalClassifiedSnapshot {
confidence: number;
intent: string;
source: ClassifiedIntent["source"];
}
export interface EvalCaseResult {
id: string;
lineNumber: number;
name?: string | undefined;
latencyMs: number;
classified: EvalClassifiedSnapshot;
derived: EvalDerivedSnapshot;
request: AgentRequest;
response: AgentResponse;
ruleJudge: RuleJudgeResult;
llmJudge: LlmJudgeResult;
error?: string | undefined;
}
export interface EvalReportSummary {
averageLatencyMs: number;
cases: number;
emptyResponses: number;
llmAverageAccuracy: number | null;
llmAverageConcision: number | null;
llmAverageFriendliness: number | null;
llmAverageOverall: number | null;
llmAverageScope: number | null;
llmJudgedCases: number;
passRate: number;
passedCases: number;
ruleAverageScore: number;
}
export interface EvalSummaryDelta {
averageLatencyMs: number | null;
llmAverageOverall: number | null;
passRate: number | null;
ruleAverageScore: number | null;
}
export interface EvalReport {
baselineDelta?: EvalSummaryDelta | undefined;
generatedAt: string;
inputPath: string;
setName: string;
summary: EvalReportSummary;
cases: EvalCaseResult[];
}
export interface EvalBaselineFile {
reportPath: string;
setName: string;
summary: EvalReportSummary;
updatedAt: string;
}