Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
+166
-17
@@ -1,9 +1,3 @@
|
||||
function printPhaseNotice(command: string): void {
|
||||
console.info(
|
||||
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
@@ -13,26 +7,181 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
interface ParsedArgs {
|
||||
command: string;
|
||||
flags: Map<string, string | true>;
|
||||
positionals: string[];
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ParsedArgs {
|
||||
const [command = "run", ...rest] = argv;
|
||||
const flags = new Map<string, string | true>();
|
||||
const positionals: string[] = [];
|
||||
|
||||
for (let index = 0; index < rest.length; index++) {
|
||||
const token = rest[index];
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!token.startsWith("--")) {
|
||||
positionals.push(token);
|
||||
continue;
|
||||
}
|
||||
|
||||
const withoutPrefix = token.slice(2);
|
||||
const equalsIndex = withoutPrefix.indexOf("=");
|
||||
if (equalsIndex >= 0) {
|
||||
const key = withoutPrefix.slice(0, equalsIndex);
|
||||
const value = withoutPrefix.slice(equalsIndex + 1);
|
||||
flags.set(key, value || true);
|
||||
continue;
|
||||
}
|
||||
|
||||
const next = rest[index + 1];
|
||||
if (next && !next.startsWith("--")) {
|
||||
flags.set(withoutPrefix, next);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
flags.set(withoutPrefix, true);
|
||||
}
|
||||
|
||||
return {
|
||||
command,
|
||||
flags,
|
||||
positionals,
|
||||
};
|
||||
}
|
||||
|
||||
function getFlagString(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): string | undefined {
|
||||
const value = flags.get(key);
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function hasFlag(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): boolean {
|
||||
return flags.has(key);
|
||||
}
|
||||
|
||||
function printRunSummary(report: {
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
cases: number;
|
||||
passRate: number;
|
||||
ruleAverageScore: number;
|
||||
averageLatencyMs: number;
|
||||
llmAverageOverall: number | null;
|
||||
llmJudgedCases: number;
|
||||
};
|
||||
baselineDelta?: {
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
averageLatencyMs: number | null;
|
||||
} | undefined;
|
||||
}, reportPath: string): void {
|
||||
const llmLine =
|
||||
report.summary.llmAverageOverall === null
|
||||
? "n/a"
|
||||
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
|
||||
|
||||
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
||||
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
|
||||
console.info(
|
||||
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
|
||||
);
|
||||
console.info(`[eval] llmAverageOverall=${llmLine}`);
|
||||
|
||||
if (report.baselineDelta) {
|
||||
console.info(
|
||||
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.info(`[eval] report=${reportPath}`);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
|
||||
loadEnvironmentFromNodeEnv();
|
||||
|
||||
const { getLogger, initLogger } = await import("@obs/logger");
|
||||
initLogger();
|
||||
const { captureGoldenSetScaffold } = await import("./capture.js");
|
||||
const logger = getLogger();
|
||||
|
||||
const [command = "run", arg1, arg2] = process.argv.slice(2);
|
||||
const parsed = parseArgs(process.argv.slice(2));
|
||||
const { command, flags, positionals } = parsed;
|
||||
|
||||
if (command === "capture") {
|
||||
const outputPath = await captureGoldenSetScaffold({
|
||||
limit: parsePositiveInteger(arg1),
|
||||
outputPath: arg2,
|
||||
});
|
||||
getLogger().info({ outputPath }, '[eval] "capture" completed');
|
||||
return;
|
||||
try {
|
||||
if (command === "capture") {
|
||||
const { captureGoldenSetScaffold } = await import("./capture.js");
|
||||
const outputPath = await captureGoldenSetScaffold({
|
||||
limit:
|
||||
parsePositiveInteger(getFlagString(flags, "limit")) ??
|
||||
parsePositiveInteger(positionals[0]),
|
||||
outputPath: getFlagString(flags, "output") ?? positionals[1],
|
||||
});
|
||||
logger.info({ outputPath }, '[eval] "capture" completed');
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "run") {
|
||||
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const { publishEvalReportIfPossible } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const setName = getFlagString(flags, "set") ?? "golden";
|
||||
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
|
||||
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
|
||||
|
||||
if (hasFlag(flags, "dry-run")) {
|
||||
const cases = await loadEvalCases(inputPath, { limit });
|
||||
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const { report, reportPath } = await runOnboardingEval({
|
||||
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
|
||||
inputPath,
|
||||
limit,
|
||||
outputPath: getFlagString(flags, "output"),
|
||||
setName,
|
||||
});
|
||||
const publishedReport = hasFlag(flags, "no-publish-report")
|
||||
? false
|
||||
: await publishEvalReportIfPossible(report);
|
||||
printRunSummary(report, reportPath);
|
||||
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "baseline") {
|
||||
const { updateEvalBaseline } = await import("./runner.js");
|
||||
const reportPath = getFlagString(flags, "report") ?? positionals[0];
|
||||
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
|
||||
console.info(
|
||||
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown eval command: ${command}`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ err: error },
|
||||
`[eval] "${command}" failed`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
printPhaseNotice(command);
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -0,0 +1,3 @@
|
||||
{"id":"greeting-short","name":"Greeting should stay short and welcoming","input":{"request":{"requestId":"eval_greeting_1","skill":"onboarding","chatroomId":"room_eval_1","userId":"user_eval_1","nickname":"<nick:greeting>","trigger":"user-message","userMessage":"안녕하세요"},"session":null,"conversationHistory":[]},"expected":{"intent":"greeting","responseMode":"llm","mustIncludeAny":["환영","반갑"],"mustNotInclude":["디스코드"]}}
|
||||
{"id":"status-photo-remaining","name":"Status question should answer only the remaining step","input":{"request":{"requestId":"eval_status_1","skill":"onboarding","chatroomId":"room_eval_2","userId":"user_eval_2","nickname":"프리리/95/여/D","trigger":"user-message","userMessage":"이제 뭐 남았지"},"session":{"chatroomId":"room_eval_2","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/D","nicknameVerified":true,"photoVerified":false,"status":"pending","userId":"user_eval_2"},"conversationHistory":[{"role":"user","content":"안녕하세요","timestamp":"2026-04-16T00:00:00.000Z"},{"role":"assistant","content":"환영합니다! 반갑습니다.","timestamp":"2026-04-16T00:00:01.000Z"}]},"expected":{"intent":"status-question","responseMode":"llm","mustIncludeAny":["남았","하트"],"mustNotInclude":["디스코드"]}}
|
||||
{"id":"nickname-invalid-tier","name":"Nickname invalid flow should focus on nickname correction","input":{"request":{"requestId":"eval_nick_1","skill":"onboarding","chatroomId":"room_eval_3","userId":"user_eval_3","nickname":"프리리/95/여/다이아","trigger":"nickname-invalid"},"session":{"chatroomId":"room_eval_3","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/다이아","nicknameVerified":false,"photoVerified":false,"status":"pending","userId":"user_eval_3"},"conversationHistory":[]},"expected":{"intent":"nickname-question","responseMode":"llm-with-tools","mustIncludeAny":["닉네임","티어","형식"],"mustNotInclude":["디스코드"]}}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { LlmClient } from "@llm/llm-client";
|
||||
import type {
|
||||
ChatCompletionOptions,
|
||||
ChatCompletionResult,
|
||||
ChatMessage,
|
||||
} from "@llm/types";
|
||||
import { judgeOnboardingWithLlm } from "./llm-judge.js";
|
||||
import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js";
|
||||
|
||||
class FakeLlmClient extends LlmClient {
|
||||
constructor(private readonly payload: string | null) {
|
||||
super({ endpoints: [] });
|
||||
}
|
||||
|
||||
hasEndpoints(): boolean {
|
||||
return this.payload !== null;
|
||||
}
|
||||
|
||||
chatCompletion(
|
||||
_messages: ChatMessage[],
|
||||
_options?: ChatCompletionOptions,
|
||||
): Promise<ChatCompletionResult | null> {
|
||||
if (this.payload === null) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: this.payload,
|
||||
endpoint: "http://judge.test/v1/chat/completions",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: this.payload,
|
||||
},
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const evalCase: EvalCaseDefinition = {
|
||||
id: "case-1",
|
||||
lineNumber: 1,
|
||||
input: {
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "이제 뭐 남았지",
|
||||
},
|
||||
},
|
||||
expected: {
|
||||
intent: "status-question",
|
||||
responseMode: "llm",
|
||||
},
|
||||
};
|
||||
|
||||
const derived: EvalDerivedSnapshot = {
|
||||
allowShortResponse: true,
|
||||
intent: "status-question",
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 128,
|
||||
tier: 0,
|
||||
};
|
||||
|
||||
describe("judgeOnboardingWithLlm", () => {
|
||||
it("parses a valid JSON judge response", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient(
|
||||
JSON.stringify({
|
||||
friendliness: 5,
|
||||
accuracy: 4,
|
||||
concision: 5,
|
||||
scope: 4,
|
||||
rationale: "질문 범위에 맞게 짧고 정확하게 답했다.",
|
||||
}),
|
||||
),
|
||||
evalCase,
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
derived,
|
||||
);
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.scores.overall).toBe(4.5);
|
||||
expect(result.provider).toBe("cliproxy");
|
||||
}
|
||||
});
|
||||
|
||||
it("skips when the judge returns invalid JSON", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient('{"friendliness":"bad"}'),
|
||||
evalCase,
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
derived,
|
||||
);
|
||||
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) {
|
||||
expect(result.reason).toBe("invalid-judge-json");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLlmJudgeMaxTokens, getLlmJudgeModel } from "../../config.js";
|
||||
import type {
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
LlmJudgeResult,
|
||||
LlmJudgeScores,
|
||||
} from "../types.js";
|
||||
|
||||
const llmJudgeSchema = z.object({
|
||||
friendliness: z.number().int().min(1).max(5),
|
||||
accuracy: z.number().int().min(1).max(5),
|
||||
concision: z.number().int().min(1).max(5),
|
||||
scope: z.number().int().min(1).max(5),
|
||||
rationale: z.string().trim().min(1).max(200),
|
||||
});
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function buildJudgePrompt(
|
||||
evalCase: EvalCaseDefinition,
|
||||
responseText: string,
|
||||
derived: EvalDerivedSnapshot,
|
||||
): string {
|
||||
return [
|
||||
"다음 온보딩 에이전트 응답을 평가해 주세요.",
|
||||
"평가 축은 friendliness, accuracy, concision, scope 입니다.",
|
||||
"각 점수는 1~5의 정수만 사용하세요.",
|
||||
"scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.",
|
||||
"",
|
||||
"[request]",
|
||||
JSON.stringify(evalCase.input.request),
|
||||
"",
|
||||
"[expected]",
|
||||
JSON.stringify({
|
||||
intent: evalCase.expected.intent ?? null,
|
||||
mustIncludeAny: evalCase.expected.mustIncludeAny ?? [],
|
||||
mustNotInclude: evalCase.expected.mustNotInclude ?? [],
|
||||
responseMode: evalCase.expected.responseMode ?? null,
|
||||
}),
|
||||
"",
|
||||
"[derived]",
|
||||
JSON.stringify(derived),
|
||||
"",
|
||||
"[assistant_response]",
|
||||
responseText,
|
||||
"",
|
||||
'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function judgeOnboardingWithLlm(
|
||||
llmClient: LlmClient,
|
||||
evalCase: EvalCaseDefinition,
|
||||
responseText: string,
|
||||
derived: EvalDerivedSnapshot,
|
||||
): Promise<LlmJudgeResult> {
|
||||
if (!responseText.trim()) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "empty-response",
|
||||
};
|
||||
}
|
||||
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "no-llm-endpoint",
|
||||
};
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildJudgePrompt(evalCase, responseText, derived),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: getLlmJudgeMaxTokens(),
|
||||
modelOverride: getLlmJudgeModel(),
|
||||
temperature: 0,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "judge-call-failed",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = llmJudgeSchema.parse(JSON.parse(response.content));
|
||||
const scores: LlmJudgeScores = {
|
||||
friendliness: parsed.friendliness,
|
||||
accuracy: parsed.accuracy,
|
||||
concision: parsed.concision,
|
||||
scope: parsed.scope,
|
||||
overall: round(
|
||||
(parsed.friendliness +
|
||||
parsed.accuracy +
|
||||
parsed.concision +
|
||||
parsed.scope) /
|
||||
4,
|
||||
),
|
||||
};
|
||||
|
||||
return {
|
||||
skipped: false,
|
||||
model: getLlmJudgeModel(),
|
||||
endpoint: response.endpoint,
|
||||
provider: response.provider,
|
||||
scores,
|
||||
rationale: parsed.rationale,
|
||||
raw: response.content,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "invalid-judge-json",
|
||||
raw: response.content,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { judgeOnboardingRuleCase } from "./rule-judge.js";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
function makeDerived(
|
||||
overrides?: Partial<DerivedState>,
|
||||
): DerivedState {
|
||||
return {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("judgeOnboardingRuleCase", () => {
|
||||
it("passes when active checks match", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
{
|
||||
intent: "status-question",
|
||||
responseMode: "llm",
|
||||
mustIncludeAny: ["방 하트 인증"],
|
||||
mustNotInclude: ["디스코드 서버"],
|
||||
},
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.score).toBe(1);
|
||||
expect(result.validatorStatus).toBe("pass");
|
||||
});
|
||||
|
||||
it("fails intent and mustNotInclude mismatches", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
{
|
||||
intent: "help-how",
|
||||
mustNotInclude: ["디스코드"],
|
||||
},
|
||||
"디스코드 서버에 먼저 들어오시면 돼요.",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find((check) => check.name === "intent")?.passed).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
result.checks.find((check) => check.name === "mustNotInclude")?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats rejecting validator results as a failed rule check", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
{},
|
||||
"디스코드 서버에 들어가시면 돼요.",
|
||||
makeDerived({
|
||||
intent: { kind: "photo-question" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.validatorStatus).toBe("reject");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
export function judgeOnboardingRuleCase(
|
||||
expected: EvalCaseExpected,
|
||||
responseText: string,
|
||||
derived: DerivedState,
|
||||
): RuleJudgeResult {
|
||||
const validation = validateOnboardingResponse(responseText, derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
});
|
||||
const checks: RuleJudgeCheck[] = [];
|
||||
|
||||
checks.push({
|
||||
name: "validator",
|
||||
passed: validation.status !== "reject",
|
||||
skipped: false,
|
||||
detail:
|
||||
validation.reasons.length > 0
|
||||
? validation.reasons.join(", ")
|
||||
: validation.status,
|
||||
});
|
||||
|
||||
const expectedIntent = expected.intent?.trim();
|
||||
if (expectedIntent) {
|
||||
checks.push({
|
||||
name: "intent",
|
||||
passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind),
|
||||
skipped: false,
|
||||
detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "intent",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const expectedResponseMode = expected.responseMode?.trim();
|
||||
if (expectedResponseMode) {
|
||||
checks.push({
|
||||
name: "responseMode",
|
||||
passed:
|
||||
normalizeText(expectedResponseMode) ===
|
||||
normalizeText(derived.responseMode),
|
||||
skipped: false,
|
||||
detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "responseMode",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
|
||||
if (mustIncludeAny.length > 0) {
|
||||
const normalizedResponse = normalizeText(responseText);
|
||||
const matched = mustIncludeAny.find((candidate) =>
|
||||
normalizedResponse.includes(normalizeText(candidate)),
|
||||
);
|
||||
|
||||
checks.push({
|
||||
name: "mustIncludeAny",
|
||||
passed: Boolean(matched),
|
||||
skipped: false,
|
||||
detail: matched
|
||||
? `matched=${matched}`
|
||||
: `expected one of: ${mustIncludeAny.join(", ")}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "mustIncludeAny",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
|
||||
if (mustNotInclude.length > 0) {
|
||||
const normalizedResponse = normalizeText(responseText);
|
||||
const matched = mustNotInclude.find((candidate) =>
|
||||
normalizedResponse.includes(normalizeText(candidate)),
|
||||
);
|
||||
|
||||
checks.push({
|
||||
name: "mustNotInclude",
|
||||
passed: !matched,
|
||||
skipped: false,
|
||||
detail: matched ? `forbidden=${matched}` : undefined,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "mustNotInclude",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const activeChecks = checks.filter((check) => !check.skipped);
|
||||
const passedChecks = activeChecks.filter((check) => check.passed).length;
|
||||
|
||||
return {
|
||||
passed: activeChecks.every((check) => check.passed),
|
||||
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
||||
checks,
|
||||
validatorStatus: validation.status,
|
||||
validatorReasons: validation.reasons,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||
import { publishEvalReportIfPossible } from "./runner.js";
|
||||
import type { EvalReport } from "./types.js";
|
||||
|
||||
const natsMocks = vi.hoisted(() => ({
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
connect: vi.fn().mockResolvedValue(undefined),
|
||||
publish: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@common/nats", () => {
|
||||
return {
|
||||
NatsClient: class {
|
||||
connect = natsMocks.connect;
|
||||
publish = natsMocks.publish;
|
||||
close = natsMocks.close;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const report: EvalReport = {
|
||||
generatedAt: "2026-04-17T00:00:00.000Z",
|
||||
inputPath: "/tmp/onboarding.jsonl",
|
||||
setName: "golden",
|
||||
summary: {
|
||||
averageLatencyMs: 100,
|
||||
cases: 1,
|
||||
emptyResponses: 0,
|
||||
llmAverageAccuracy: 4,
|
||||
llmAverageConcision: 4,
|
||||
llmAverageFriendliness: 5,
|
||||
llmAverageOverall: 4.25,
|
||||
llmAverageScope: 4,
|
||||
llmJudgedCases: 1,
|
||||
passRate: 1,
|
||||
passedCases: 1,
|
||||
ruleAverageScore: 1,
|
||||
},
|
||||
cases: [],
|
||||
};
|
||||
|
||||
describe("publishEvalReportIfPossible", () => {
|
||||
it("publishes the report to the eval report subject", async () => {
|
||||
const result = await publishEvalReportIfPossible(report);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(natsMocks.connect).toHaveBeenCalledTimes(1);
|
||||
expect(natsMocks.publish).toHaveBeenCalledWith(
|
||||
AGENT_EVAL_REPORT_SUBJECT,
|
||||
report,
|
||||
);
|
||||
expect(natsMocks.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,842 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, resolve } from "node:path";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import { GarnetClient } from "@common/garnet";
|
||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||
import { NatsClient } from "@common/nats";
|
||||
import { getConversationSummaryKey } from "@conv/conversation-store";
|
||||
import { LlmClient } from "@llm/llm-client";
|
||||
import type { ProviderName } from "@llm/types";
|
||||
import { runWithContext } from "@obs/context";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { RoomMessage } from "@room/types";
|
||||
import { RoomContextService } from "@room/room-context-service";
|
||||
import { OnboardingSkill } from "@skills/onboarding/index";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import { OnboardingIntentClassifier } from "@skills/onboarding/state/classifier";
|
||||
import { deriveState } from "@skills/onboarding/state/derive";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl, getOpenRouterEnabled } from "../config.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeOnboardingRuleCase } from "./judges/rule-judge.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalCaseExpected,
|
||||
EvalCaseInput,
|
||||
EvalCaseResult,
|
||||
EvalClassifiedSnapshot,
|
||||
EvalDerivedSnapshot,
|
||||
EvalReport,
|
||||
EvalReportSummary,
|
||||
EvalSummaryDelta,
|
||||
} from "./types.js";
|
||||
|
||||
const DEFAULT_REPORTS_DIR = "eval-reports";
|
||||
const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
|
||||
const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl";
|
||||
const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl";
|
||||
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
||||
|
||||
class InMemoryGarnetClient extends GarnetClient {
|
||||
private readonly hashStore = new Map<string, Record<string, string>>();
|
||||
private readonly stringStore = new Map<string, string>();
|
||||
|
||||
getJson<T>(key: string): Promise<T | null> {
|
||||
const raw = this.stringStore.get(key);
|
||||
return Promise.resolve(raw ? (JSON.parse(raw) as T) : null);
|
||||
}
|
||||
|
||||
setJson(
|
||||
key: string,
|
||||
value: unknown,
|
||||
_ttlSeconds?: number,
|
||||
): Promise<void> {
|
||||
this.stringStore.set(key, JSON.stringify(value));
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
getString(key: string): Promise<string | null> {
|
||||
return Promise.resolve(this.stringStore.get(key) ?? null);
|
||||
}
|
||||
|
||||
setString(
|
||||
key: string,
|
||||
value: string,
|
||||
_ttlSeconds?: number,
|
||||
): Promise<void> {
|
||||
this.stringStore.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
delete(key: string): Promise<number> {
|
||||
const stringDeleted = this.stringStore.delete(key);
|
||||
const hashDeleted = this.hashStore.delete(key);
|
||||
return Promise.resolve(stringDeleted || hashDeleted ? 1 : 0);
|
||||
}
|
||||
|
||||
getHash(key: string): Promise<Record<string, string>> {
|
||||
return Promise.resolve({ ...(this.hashStore.get(key) ?? {}) });
|
||||
}
|
||||
|
||||
setHash(
|
||||
key: string,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
this.hashStore.set(key, { ...fields });
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class InMemoryRoomContextService extends RoomContextService {
|
||||
constructor(private readonly messages: RoomMessage[]) {
|
||||
super();
|
||||
}
|
||||
|
||||
getRecentMessages(
|
||||
_chatroomId: string,
|
||||
limit?: number,
|
||||
): Promise<RoomMessage[]> {
|
||||
const effectiveLimit = limit ?? this.messages.length;
|
||||
return Promise.resolve(this.messages.slice(-effectiveLimit));
|
||||
}
|
||||
}
|
||||
|
||||
interface RawEvalCaseLine {
|
||||
expected?: Record<string, unknown>;
|
||||
id?: unknown;
|
||||
input?: Record<string, unknown>;
|
||||
name?: unknown;
|
||||
notes?: unknown;
|
||||
}
|
||||
|
||||
export interface EvalRunOptions {
|
||||
enableLlmJudge?: boolean | undefined;
|
||||
inputPath?: string | undefined;
|
||||
limit?: number | undefined;
|
||||
outputPath?: string | undefined;
|
||||
setName?: string | undefined;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function average(values: number[]): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return round(values.reduce((sum, value) => sum + value, 0) / values.length);
|
||||
}
|
||||
|
||||
function averageOrZero(values: number[]): number {
|
||||
return values.length === 0 ? 0 : round(values.reduce((sum, value) => sum + value, 0) / values.length);
|
||||
}
|
||||
|
||||
function firstNonEmpty(
|
||||
...values: Array<string | null | undefined>
|
||||
): string | undefined {
|
||||
for (const value of values) {
|
||||
if (value?.trim()) {
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseEndpointList(
|
||||
...values: Array<string | null | undefined>
|
||||
): string[] {
|
||||
const endpoints: string[] = [];
|
||||
|
||||
for (const value of values) {
|
||||
if (!value?.trim()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const item of value.split(/[\n,]/u)) {
|
||||
const trimmed = item.trim();
|
||||
if (trimmed) {
|
||||
endpoints.push(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...new Set(endpoints)];
|
||||
}
|
||||
|
||||
function toProviderEndpoints(
|
||||
urls: string[],
|
||||
provider?: ProviderName,
|
||||
): Array<string | { url: string; provider?: ProviderName | undefined }> {
|
||||
if (!provider) {
|
||||
return urls;
|
||||
}
|
||||
|
||||
return urls.map((url) => ({ url, provider }));
|
||||
}
|
||||
|
||||
function createEvalLlmClient(): LlmClient {
|
||||
const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]);
|
||||
const llamaEndpoints = parseEndpointList(
|
||||
process.env["AGENT_LLM_API_URLS"],
|
||||
process.env["LLAMA_SERVER_API_URL"],
|
||||
);
|
||||
const openRouterEndpoints = getOpenRouterEnabled()
|
||||
? parseEndpointList(process.env["OPENROUTER_API_URL"], getOpenRouterApiUrl())
|
||||
: [];
|
||||
|
||||
const llmApiKey = firstNonEmpty(
|
||||
process.env["AGENT_LLM_API_KEY"],
|
||||
process.env["LLAMA_SERVER_API_KEY"],
|
||||
);
|
||||
const providerApiKeys = Object.fromEntries(
|
||||
[
|
||||
["cliproxy", firstNonEmpty(process.env["CLI_PROXY_API_KEY"])],
|
||||
["llama", llmApiKey],
|
||||
["openrouter", firstNonEmpty(process.env["OPENROUTER_API_KEY"])],
|
||||
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
||||
) as Partial<Record<ProviderName, string>>;
|
||||
|
||||
return new LlmClient({
|
||||
endpoints: [
|
||||
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
|
||||
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
|
||||
...toProviderEndpoints(llamaEndpoints, "llama"),
|
||||
],
|
||||
apiKey: llmApiKey,
|
||||
model: firstNonEmpty(process.env["AGENT_LLM_MODEL"]),
|
||||
providerApiKeys,
|
||||
});
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
function getStringField(record: Record<string, unknown>, key: string): string | undefined {
|
||||
const value = record[key];
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function getStringArrayField(
|
||||
record: Record<string, unknown>,
|
||||
key: string,
|
||||
): string[] | undefined {
|
||||
const value = record[key];
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const strings = value.filter((item): item is string => typeof item === "string");
|
||||
return strings.length > 0 ? strings : [];
|
||||
}
|
||||
|
||||
function toAgentRequest(raw: Record<string, unknown>): AgentRequest {
|
||||
const requestId = getStringField(raw, "requestId");
|
||||
const skill = getStringField(raw, "skill");
|
||||
const chatroomId = getStringField(raw, "chatroomId");
|
||||
const userId = getStringField(raw, "userId");
|
||||
const nickname = getStringField(raw, "nickname");
|
||||
const trigger = getStringField(raw, "trigger");
|
||||
|
||||
if (!requestId || !skill || !chatroomId || !userId || !nickname || !trigger) {
|
||||
throw new Error("input.request is missing required AgentRequest fields");
|
||||
}
|
||||
|
||||
const request: AgentRequest = {
|
||||
requestId,
|
||||
skill,
|
||||
chatroomId,
|
||||
userId,
|
||||
nickname,
|
||||
trigger,
|
||||
};
|
||||
|
||||
const userMessage = getStringField(raw, "userMessage");
|
||||
if (userMessage !== undefined) {
|
||||
request.userMessage = userMessage;
|
||||
}
|
||||
|
||||
const context = raw["context"];
|
||||
if (isRecord(context)) {
|
||||
request.context = context;
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
function parseConversationHistory(raw: unknown): EvalCaseInput["conversationHistory"] {
|
||||
if (!Array.isArray(raw)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return raw
|
||||
.filter((item): item is Record<string, unknown> => isRecord(item))
|
||||
.map((item) => {
|
||||
const role = item["role"];
|
||||
const content = getStringField(item, "content") ?? "";
|
||||
const timestamp = getStringField(item, "timestamp") ?? new Date(0).toISOString();
|
||||
|
||||
if (role !== "user" && role !== "assistant") {
|
||||
throw new Error("conversationHistory contains an invalid role");
|
||||
}
|
||||
|
||||
return {
|
||||
role,
|
||||
content,
|
||||
timestamp,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function parseRoomContext(raw: unknown): EvalCaseInput["roomContext"] {
|
||||
if (!Array.isArray(raw)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return raw
|
||||
.filter((item): item is Record<string, unknown> => isRecord(item))
|
||||
.map((item) => ({
|
||||
senderId: getStringField(item, "senderId") ?? "",
|
||||
nickname: getStringField(item, "nickname") ?? "알 수 없음",
|
||||
message: getStringField(item, "message") ?? "",
|
||||
timestamp: getStringField(item, "timestamp") ?? new Date(0).toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpected {
|
||||
if (!raw) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const expected: EvalCaseExpected = {};
|
||||
const intent = getStringField(raw, "intent");
|
||||
const responseMode = getStringField(raw, "responseMode");
|
||||
const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny");
|
||||
const mustNotInclude = getStringArrayField(raw, "mustNotInclude");
|
||||
|
||||
if (intent) {
|
||||
expected.intent = intent;
|
||||
}
|
||||
if (responseMode) {
|
||||
expected.responseMode = responseMode;
|
||||
}
|
||||
if (mustIncludeAny) {
|
||||
expected.mustIncludeAny = mustIncludeAny;
|
||||
}
|
||||
if (mustNotInclude) {
|
||||
expected.mustNotInclude = mustNotInclude;
|
||||
}
|
||||
|
||||
return expected;
|
||||
}
|
||||
|
||||
function parseEvalCaseLine(
|
||||
raw: RawEvalCaseLine,
|
||||
lineNumber: number,
|
||||
sourcePath: string,
|
||||
): EvalCaseDefinition {
|
||||
if (!isRecord(raw.input)) {
|
||||
throw new Error("input is required");
|
||||
}
|
||||
|
||||
const requestRaw = raw.input["request"];
|
||||
if (!isRecord(requestRaw)) {
|
||||
throw new Error("input.request is required");
|
||||
}
|
||||
|
||||
const input: EvalCaseInput = {
|
||||
request: toAgentRequest(requestRaw),
|
||||
};
|
||||
|
||||
const conversationHistory = parseConversationHistory(raw.input["conversationHistory"]);
|
||||
if (conversationHistory) {
|
||||
input.conversationHistory = conversationHistory;
|
||||
}
|
||||
|
||||
const roomContext = parseRoomContext(raw.input["roomContext"]);
|
||||
if (roomContext) {
|
||||
input.roomContext = roomContext;
|
||||
}
|
||||
|
||||
if (raw.input["summary"] !== undefined) {
|
||||
const summary = getStringField(raw.input, "summary");
|
||||
if (summary !== undefined) {
|
||||
input.summary = summary;
|
||||
}
|
||||
}
|
||||
|
||||
if (raw.input["session"] === null || isRecord(raw.input["session"])) {
|
||||
input.session = (raw.input["session"] ?? null) as OnboardingSession | null;
|
||||
}
|
||||
|
||||
if (raw.input["blackboard"] === null || isRecord(raw.input["blackboard"])) {
|
||||
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
|
||||
}
|
||||
|
||||
return {
|
||||
id:
|
||||
typeof raw.id === "string" && raw.id.trim()
|
||||
? raw.id.trim()
|
||||
: `${basename(sourcePath)}:${lineNumber}`,
|
||||
lineNumber,
|
||||
name: typeof raw.name === "string" ? raw.name : undefined,
|
||||
notes: typeof raw.notes === "string" ? raw.notes : undefined,
|
||||
input,
|
||||
expected: parseExpected(isRecord(raw.expected) ? raw.expected : undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
||||
tier: derived.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function toClassifiedSnapshot(classified: ClassifiedIntent): EvalClassifiedSnapshot {
|
||||
return {
|
||||
confidence: round(classified.confidence),
|
||||
intent: classified.intent.kind,
|
||||
source: classified.source,
|
||||
};
|
||||
}
|
||||
|
||||
function toRoomMessages(
|
||||
input: EvalCaseInput["roomContext"],
|
||||
): RoomMessage[] {
|
||||
return (
|
||||
input?.map((message) => ({
|
||||
senderId: message.senderId,
|
||||
nickname: message.nickname,
|
||||
message: message.message,
|
||||
timestamp: new Date(message.timestamp),
|
||||
})) ?? []
|
||||
);
|
||||
}
|
||||
|
||||
async function hydrateEvalState(
|
||||
garnetClient: InMemoryGarnetClient,
|
||||
input: EvalCaseInput,
|
||||
): Promise<void> {
|
||||
const {
|
||||
request: { chatroomId, userId },
|
||||
} = input;
|
||||
|
||||
if (input.session !== undefined) {
|
||||
await garnetClient.setJson(
|
||||
`${getOnboardingSessionKeyPrefix()}:onboarding:${chatroomId}:${userId}`,
|
||||
input.session,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.blackboard !== undefined) {
|
||||
await garnetClient.setJson(
|
||||
`lolympus-agent:bb:${chatroomId}:${userId}`,
|
||||
input.blackboard,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.summary?.trim()) {
|
||||
await garnetClient.setString(
|
||||
getConversationSummaryKey(chatroomId, userId),
|
||||
input.summary,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
||||
const llmResults = results.flatMap((result) =>
|
||||
result.llmJudge.skipped ? [] : [result.llmJudge],
|
||||
);
|
||||
|
||||
return {
|
||||
averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)),
|
||||
cases: results.length,
|
||||
emptyResponses: results.filter((result) => !result.response.text.trim()).length,
|
||||
llmAverageAccuracy: average(
|
||||
llmResults.map((result) => result.scores.accuracy),
|
||||
),
|
||||
llmAverageConcision: average(
|
||||
llmResults.map((result) => result.scores.concision),
|
||||
),
|
||||
llmAverageFriendliness: average(
|
||||
llmResults.map((result) => result.scores.friendliness),
|
||||
),
|
||||
llmAverageOverall: average(
|
||||
llmResults.map((result) => result.scores.overall),
|
||||
),
|
||||
llmAverageScope: average(
|
||||
llmResults.map((result) => result.scores.scope),
|
||||
),
|
||||
llmJudgedCases: llmResults.length,
|
||||
passRate:
|
||||
results.length === 0
|
||||
? 0
|
||||
: round(
|
||||
results.filter((result) => result.ruleJudge.passed).length /
|
||||
results.length,
|
||||
),
|
||||
passedCases: results.filter((result) => result.ruleJudge.passed).length,
|
||||
ruleAverageScore: averageOrZero(
|
||||
results.map((result) => result.ruleJudge.score),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function createBaselineDelta(
|
||||
current: EvalReportSummary,
|
||||
baseline: EvalReportSummary,
|
||||
): EvalSummaryDelta {
|
||||
const subtract = (
|
||||
left: number | null,
|
||||
right: number | null,
|
||||
): number | null => {
|
||||
if (left === null || right === null) {
|
||||
return null;
|
||||
}
|
||||
return round(left - right);
|
||||
};
|
||||
|
||||
return {
|
||||
averageLatencyMs: subtract(current.averageLatencyMs, baseline.averageLatencyMs),
|
||||
llmAverageOverall: subtract(
|
||||
current.llmAverageOverall,
|
||||
baseline.llmAverageOverall,
|
||||
),
|
||||
passRate: subtract(current.passRate, baseline.passRate),
|
||||
ruleAverageScore: subtract(current.ruleAverageScore, baseline.ruleAverageScore),
|
||||
};
|
||||
}
|
||||
|
||||
function buildDefaultReportPath(timestamp: string): string {
|
||||
const filename = `${timestamp.replace(/[:.]/gu, "-")}.json`;
|
||||
return resolve(process.cwd(), DEFAULT_REPORTS_DIR, filename);
|
||||
}
|
||||
|
||||
async function writeReport(
|
||||
report: EvalReport,
|
||||
outputPath: string,
|
||||
): Promise<string> {
|
||||
const resolved = resolve(process.cwd(), outputPath);
|
||||
await mkdir(dirname(resolved), { recursive: true });
|
||||
await writeFile(resolved, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function loadBaselineFile(): Promise<EvalBaselineFile | null> {
|
||||
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
|
||||
if (!existsSync(baselinePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const raw = await readFile(baselinePath, "utf8");
|
||||
return JSON.parse(raw) as EvalBaselineFile;
|
||||
}
|
||||
|
||||
export function resolveEvalSetPath(setNameOrPath = "golden"): string {
|
||||
if (setNameOrPath === "golden") {
|
||||
return resolve(process.cwd(), GOLDEN_SET_PATH);
|
||||
}
|
||||
|
||||
if (setNameOrPath === "capture") {
|
||||
return resolve(process.cwd(), CAPTURE_SET_PATH);
|
||||
}
|
||||
|
||||
if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) {
|
||||
return resolve(process.cwd(), setNameOrPath);
|
||||
}
|
||||
|
||||
return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`);
|
||||
}
|
||||
|
||||
export async function loadEvalCases(
|
||||
inputPath: string,
|
||||
options?: { limit?: number | undefined },
|
||||
): Promise<EvalCaseDefinition[]> {
|
||||
const resolvedPath = resolve(process.cwd(), inputPath);
|
||||
if (!existsSync(resolvedPath)) {
|
||||
throw new Error(`Eval set not found: ${resolvedPath}`);
|
||||
}
|
||||
|
||||
const raw = await readFile(resolvedPath, "utf8");
|
||||
const lines = raw.split(/\r?\n/u).filter((line) => line.trim());
|
||||
const cases = lines.map((line, index) =>
|
||||
parseEvalCaseLine(
|
||||
JSON.parse(line) as RawEvalCaseLine,
|
||||
index + 1,
|
||||
resolvedPath,
|
||||
),
|
||||
);
|
||||
|
||||
return options?.limit ? cases.slice(0, options.limit) : cases;
|
||||
}
|
||||
|
||||
async function evaluateCase(
|
||||
evalCase: EvalCaseDefinition,
|
||||
llmClient: LlmClient,
|
||||
enableLlmJudge: boolean,
|
||||
): Promise<EvalCaseResult> {
|
||||
const input = evalCase.input;
|
||||
const inMemoryGarnet = new InMemoryGarnetClient();
|
||||
await hydrateEvalState(inMemoryGarnet, input);
|
||||
|
||||
const roomContextService =
|
||||
input.roomContext && input.roomContext.length > 0
|
||||
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
|
||||
: undefined;
|
||||
const skill = new OnboardingSkill(
|
||||
inMemoryGarnet,
|
||||
llmClient,
|
||||
roomContextService,
|
||||
noopShadowEvalPublisher,
|
||||
);
|
||||
const classifier = new OnboardingIntentClassifier(llmClient);
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
chatroomId: input.request.chatroomId,
|
||||
deadlineAt: new Date(Date.now() + 4_500).toISOString(),
|
||||
requestId: input.request.requestId,
|
||||
skill: input.request.skill,
|
||||
startedAt: new Date().toISOString(),
|
||||
trigger: input.request.trigger,
|
||||
userId: input.request.userId,
|
||||
},
|
||||
async () => {
|
||||
const classified = await classifier.classify(
|
||||
input.request.userMessage,
|
||||
input.request.trigger,
|
||||
);
|
||||
const derived = deriveState(
|
||||
input.session ?? null,
|
||||
input.request.trigger,
|
||||
classified.intent,
|
||||
);
|
||||
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const response = await skill.handle(
|
||||
input.request,
|
||||
input.conversationHistory ?? [],
|
||||
);
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const derivedSnapshot = toDerivedSnapshot(derived);
|
||||
const ruleJudge = judgeOnboardingRuleCase(
|
||||
evalCase.expected,
|
||||
response.text,
|
||||
derived,
|
||||
);
|
||||
const llmJudge = enableLlmJudge
|
||||
? await judgeOnboardingWithLlm(
|
||||
llmClient,
|
||||
evalCase,
|
||||
response.text,
|
||||
derivedSnapshot,
|
||||
)
|
||||
: {
|
||||
skipped: true as const,
|
||||
reason: "disabled",
|
||||
};
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
name: evalCase.name,
|
||||
latencyMs,
|
||||
classified: toClassifiedSnapshot(classified),
|
||||
derived: derivedSnapshot,
|
||||
request: input.request,
|
||||
response,
|
||||
ruleJudge,
|
||||
llmJudge,
|
||||
};
|
||||
} catch (error) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
name: evalCase.name,
|
||||
latencyMs,
|
||||
classified: toClassifiedSnapshot(classified),
|
||||
derived: toDerivedSnapshot(derived),
|
||||
request: input.request,
|
||||
response: {
|
||||
requestId: input.request.requestId,
|
||||
text: "",
|
||||
confidence: 0,
|
||||
source: "template",
|
||||
metadata: {
|
||||
error: errorMessage,
|
||||
},
|
||||
},
|
||||
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
reason: "skill-error",
|
||||
},
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function runOnboardingEval(
|
||||
options?: EvalRunOptions,
|
||||
): Promise<{ report: EvalReport; reportPath: string }> {
|
||||
const logger = getLogger();
|
||||
const setName = options?.setName ?? "golden";
|
||||
const inputPath = options?.inputPath
|
||||
? resolve(process.cwd(), options.inputPath)
|
||||
: resolveEvalSetPath(setName);
|
||||
const evalCases = await loadEvalCases(inputPath, {
|
||||
limit: options?.limit,
|
||||
});
|
||||
|
||||
const llmClient = createEvalLlmClient();
|
||||
const enableLlmJudge = options?.enableLlmJudge ?? true;
|
||||
const caseResults: EvalCaseResult[] = [];
|
||||
|
||||
for (const evalCase of evalCases) {
|
||||
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge));
|
||||
}
|
||||
|
||||
const summary = createSummary(caseResults);
|
||||
const baseline = await loadBaselineFile();
|
||||
const report: EvalReport = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
inputPath,
|
||||
setName,
|
||||
summary,
|
||||
cases: caseResults,
|
||||
};
|
||||
|
||||
if (baseline) {
|
||||
report.baselineDelta = createBaselineDelta(summary, baseline.summary);
|
||||
}
|
||||
|
||||
const reportPath = await writeReport(
|
||||
report,
|
||||
options?.outputPath ?? buildDefaultReportPath(report.generatedAt),
|
||||
);
|
||||
logger.info(
|
||||
{
|
||||
cases: summary.cases,
|
||||
passRate: summary.passRate,
|
||||
reportPath,
|
||||
},
|
||||
"[Eval] run completed",
|
||||
);
|
||||
|
||||
return { report, reportPath };
|
||||
}
|
||||
|
||||
export async function findLatestEvalReportPath(): Promise<string | null> {
|
||||
const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR);
|
||||
if (!existsSync(reportsDir)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const entries = await readdir(reportsDir, { withFileTypes: true });
|
||||
const candidates = entries
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => entry.name)
|
||||
.filter((name) => name.endsWith(".json") && name !== "baseline.json")
|
||||
.sort((left, right) => right.localeCompare(left));
|
||||
|
||||
if (candidates.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const latest = candidates[0];
|
||||
if (!latest) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return resolve(reportsDir, latest);
|
||||
}
|
||||
|
||||
export async function updateEvalBaseline(
|
||||
reportPath?: string,
|
||||
): Promise<{
|
||||
baseline: EvalBaselineFile;
|
||||
baselinePath: string;
|
||||
storedInRedis: boolean;
|
||||
}> {
|
||||
const sourcePath =
|
||||
reportPath?.trim() ? resolve(process.cwd(), reportPath) : await findLatestEvalReportPath();
|
||||
if (!sourcePath) {
|
||||
throw new Error("No eval report found to baseline");
|
||||
}
|
||||
|
||||
const raw = await readFile(sourcePath, "utf8");
|
||||
const report = JSON.parse(raw) as EvalReport;
|
||||
const baseline: EvalBaselineFile = {
|
||||
reportPath: sourcePath,
|
||||
setName: report.setName,
|
||||
summary: report.summary,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
|
||||
await mkdir(dirname(baselinePath), { recursive: true });
|
||||
await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
|
||||
|
||||
const storedInRedis = await persistBaselineInRedis(baseline);
|
||||
|
||||
return { baseline, baselinePath, storedInRedis };
|
||||
}
|
||||
|
||||
async function persistBaselineInRedis(
|
||||
baseline: EvalBaselineFile,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const { GarnetClient } = await import("@common/garnet");
|
||||
const garnetClient = new GarnetClient();
|
||||
await garnetClient.connect();
|
||||
try {
|
||||
await garnetClient.setString(
|
||||
EVAL_BASELINE_KEY,
|
||||
JSON.stringify(baseline),
|
||||
);
|
||||
return true;
|
||||
} finally {
|
||||
await garnetClient.close();
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function publishEvalReportIfPossible(
|
||||
report: EvalReport,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const natsClient = new NatsClient();
|
||||
await natsClient.connect();
|
||||
try {
|
||||
await natsClient.publish(AGENT_EVAL_REPORT_SUBJECT, report);
|
||||
return true;
|
||||
} finally {
|
||||
await natsClient.close();
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
ShadowEvalBus,
|
||||
shouldSampleShadowEval,
|
||||
} from "./shadow-bus.js";
|
||||
|
||||
const request: AgentRequest = {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "이제 뭐 남았지",
|
||||
};
|
||||
|
||||
const response: AgentResponse = {
|
||||
requestId: "req-1",
|
||||
text: "이제 방 하트 인증만 남았어요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
};
|
||||
|
||||
const classified: ClassifiedIntent = {
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.98,
|
||||
source: "regex",
|
||||
};
|
||||
|
||||
const derived: DerivedState = {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
};
|
||||
|
||||
describe("shouldSampleShadowEval", () => {
|
||||
it("samples when the random value is under the tier rate", () => {
|
||||
expect(shouldSampleShadowEval(0, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not sample when the random value exceeds the tier rate", () => {
|
||||
expect(shouldSampleShadowEval(0, 1)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ShadowEvalBus", () => {
|
||||
it("publishes a shadow-eval stream entry when sampled", () => {
|
||||
const appendStream = vi.fn().mockResolvedValue("1-0");
|
||||
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
|
||||
|
||||
new ShadowEvalBus({ appendStream }).publish({
|
||||
classified,
|
||||
createdAt: "2026-04-16T00:00:00.000Z",
|
||||
derived,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
|
||||
expect(appendStream).toHaveBeenCalledTimes(1);
|
||||
expect(appendStream.mock.calls[0]?.[0]).toBe("lolympus-agent:background");
|
||||
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
|
||||
type: "shadow-eval",
|
||||
requestId: "req-1",
|
||||
tier: "0",
|
||||
});
|
||||
|
||||
randomSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("skips publishing for empty responses", () => {
|
||||
const appendStream = vi.fn().mockResolvedValue("1-0");
|
||||
|
||||
new ShadowEvalBus({ appendStream }).publish({
|
||||
classified,
|
||||
createdAt: "2026-04-16T00:00:00.000Z",
|
||||
derived,
|
||||
request,
|
||||
response: {
|
||||
...response,
|
||||
text: "",
|
||||
},
|
||||
});
|
||||
|
||||
expect(appendStream).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
getBackgroundStreamMaxLen,
|
||||
getShadowEvalSamplingRate,
|
||||
} from "../config.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
|
||||
export interface ShadowEvalEnvelope {
|
||||
classified: ClassifiedIntent;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
|
||||
export interface ShadowEvalPublisher {
|
||||
publish(envelope: ShadowEvalEnvelope): void;
|
||||
}
|
||||
|
||||
export const noopShadowEvalPublisher: ShadowEvalPublisher = {
|
||||
publish(): void {},
|
||||
};
|
||||
|
||||
export function shouldSampleShadowEval(
|
||||
tier: 0 | 1 | 2,
|
||||
randomValue = Math.random(),
|
||||
): boolean {
|
||||
return randomValue < getShadowEvalSamplingRate(tier);
|
||||
}
|
||||
|
||||
export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
) {}
|
||||
|
||||
publish(envelope: ShadowEvalEnvelope): void {
|
||||
if (!envelope.response.text.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!shouldSampleShadowEval(envelope.derived.tier)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const logger = getRequestLogger();
|
||||
|
||||
void this.garnetClient
|
||||
.appendStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval",
|
||||
requestId: envelope.request.requestId,
|
||||
chatroomId: envelope.request.chatroomId,
|
||||
userId: envelope.request.userId,
|
||||
tier: String(envelope.derived.tier),
|
||||
latencyMs:
|
||||
envelope.latencyMs !== undefined
|
||||
? String(envelope.latencyMs)
|
||||
: "",
|
||||
input: JSON.stringify({
|
||||
request: envelope.request,
|
||||
}),
|
||||
output: JSON.stringify(envelope.response),
|
||||
derived: JSON.stringify(envelope.derived),
|
||||
classified: JSON.stringify(envelope.classified),
|
||||
createdAt: envelope.createdAt,
|
||||
},
|
||||
{
|
||||
maxLen: getBackgroundStreamMaxLen(),
|
||||
},
|
||||
)
|
||||
.catch((error) => {
|
||||
logger.warn(
|
||||
{
|
||||
err: error,
|
||||
requestId: envelope.request.requestId,
|
||||
},
|
||||
"[ShadowEvalBus] Failed to publish shadow eval job",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
calculateEvalScoreDropPct,
|
||||
calculateLlmErrorRate,
|
||||
calculatePercentile,
|
||||
} from "./shadow-worker.js";
|
||||
|
||||
describe("calculatePercentile", () => {
|
||||
it("returns null for empty input", () => {
|
||||
expect(calculatePercentile([], 0.95)).toBeNull();
|
||||
});
|
||||
|
||||
it("calculates the p95 from sorted values", () => {
|
||||
expect(calculatePercentile([100, 200, 300, 400, 500], 0.95)).toBe(500);
|
||||
});
|
||||
});
|
||||
|
||||
describe("calculateEvalScoreDropPct", () => {
|
||||
it("returns null without a valid baseline", () => {
|
||||
expect(calculateEvalScoreDropPct(4.2, null)).toBeNull();
|
||||
expect(calculateEvalScoreDropPct(4.2, 0)).toBeNull();
|
||||
});
|
||||
|
||||
it("computes percentage drop from baseline", () => {
|
||||
expect(calculateEvalScoreDropPct(3.6, 4.0)).toBeCloseTo(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe("calculateLlmErrorRate", () => {
|
||||
it("returns null when no observations are present", () => {
|
||||
expect(calculateLlmErrorRate([])).toBeNull();
|
||||
});
|
||||
|
||||
it("computes error ratio across observations", () => {
|
||||
expect(
|
||||
calculateLlmErrorRate([
|
||||
{
|
||||
llmJudgeErrored: true,
|
||||
llmOverall: null,
|
||||
recordedAt: 1,
|
||||
responseLatencyMs: 100,
|
||||
},
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4.2,
|
||||
recordedAt: 2,
|
||||
responseLatencyMs: 200,
|
||||
},
|
||||
]),
|
||||
).toBe(0.5);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,741 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type {
|
||||
GarnetClient,
|
||||
StreamEntry,
|
||||
StreamPendingEntry,
|
||||
} from "@common/garnet";
|
||||
import { summarizeConversation } from "@conv/summarizer";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { AlertSender } from "@obs/alert";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import {
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getEvalStreamMaxLen,
|
||||
} from "../config.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
} from "./types.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
|
||||
const BACKGROUND_GROUP = "bg-worker";
|
||||
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
||||
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
||||
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
|
||||
const ALERT_WINDOW_MS = 5 * 60_000;
|
||||
const ALERT_WINDOW_LABEL = "5m";
|
||||
const ALERT_TICK_MS = 60_000;
|
||||
const MAX_DELIVERIES = 3;
|
||||
const MIN_ERROR_RATE_OBSERVATIONS = 5;
|
||||
const MIN_LATENCY_OBSERVATIONS = 5;
|
||||
const MIN_SCORE_OBSERVATIONS = 3;
|
||||
const PENDING_IDLE_MS = 1_000;
|
||||
const READ_BLOCK_MS = 5_000;
|
||||
const READ_COUNT = 10;
|
||||
const PENDING_BATCH_SIZE = 20;
|
||||
const STOP_TIMEOUT_MS = 10_000;
|
||||
|
||||
type BackgroundJobType = "shadow-eval" | "summarize";
|
||||
|
||||
interface ShadowEvalJob {
|
||||
classified: ClassifiedIntent;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
|
||||
interface EvalObservation {
|
||||
llmJudgeErrored: boolean;
|
||||
llmOverall: number | null;
|
||||
recordedAt: number;
|
||||
responseLatencyMs: number | null;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function promiseWithTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
): Promise<T> {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise<T>((_, reject) => {
|
||||
setTimeout(() => {
|
||||
reject(new Error(`Timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
function getJobType(message: StreamEntry): string {
|
||||
return message.message["type"] ?? "unknown";
|
||||
}
|
||||
|
||||
function parseOptionalNumberField(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseJsonField<T>(
|
||||
value: string | undefined,
|
||||
fieldName: string,
|
||||
): T {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`Missing ${fieldName} field`);
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
||||
tier: derived.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition {
|
||||
return {
|
||||
id: `shadow:${messageId}`,
|
||||
lineNumber: 0,
|
||||
input: {
|
||||
request: job.request,
|
||||
},
|
||||
expected: {},
|
||||
};
|
||||
}
|
||||
|
||||
function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
const input = parseJsonField<{ request: AgentRequest }>(
|
||||
message.message["input"],
|
||||
"input",
|
||||
);
|
||||
|
||||
return {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
message.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
||||
derived: parseJsonField<DerivedState>(
|
||||
message.message["derived"],
|
||||
"derived",
|
||||
),
|
||||
latencyMs: parseOptionalNumberField(message.message["latencyMs"]),
|
||||
request: input.request,
|
||||
response: parseJsonField<AgentResponse>(
|
||||
message.message["output"],
|
||||
"output",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function calculatePercentile(
|
||||
values: number[],
|
||||
percentile: number,
|
||||
): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.min(
|
||||
sorted.length - 1,
|
||||
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
||||
);
|
||||
const selected = sorted[index];
|
||||
return selected === undefined ? null : selected;
|
||||
}
|
||||
|
||||
export function calculateEvalScoreDropPct(
|
||||
currentAverage: number | null,
|
||||
baselineAverage: number | null,
|
||||
): number | null {
|
||||
if (
|
||||
currentAverage === null ||
|
||||
baselineAverage === null ||
|
||||
baselineAverage <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return ((baselineAverage - currentAverage) / baselineAverage) * 100;
|
||||
}
|
||||
|
||||
export function calculateLlmErrorRate(
|
||||
observations: EvalObservation[],
|
||||
): number | null {
|
||||
if (observations.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const errorCount = observations.filter(
|
||||
(observation) => observation.llmJudgeErrored,
|
||||
).length;
|
||||
return errorCount / observations.length;
|
||||
}
|
||||
|
||||
export class ShadowWorker {
|
||||
private lastAggregateAt = 0;
|
||||
private loopPromise: Promise<void> | null = null;
|
||||
private readonly logger = getLogger();
|
||||
private readonly recentEvalObservations: EvalObservation[] = [];
|
||||
private stopRequested = false;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
private readonly llmClient: LlmClient,
|
||||
private readonly conversationStore: ConversationStore,
|
||||
private readonly alertSender?: AlertSender,
|
||||
private readonly consumerName = `bg-worker-${process.pid}`,
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.ensureConsumerGroup();
|
||||
|
||||
if (this.loopPromise) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.stopRequested = false;
|
||||
this.loopPromise = this.runLoop();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopRequested = true;
|
||||
|
||||
if (!this.loopPromise) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{ err: error },
|
||||
"[ShadowWorker] Stop timed out while waiting for the loop to finish",
|
||||
);
|
||||
} finally {
|
||||
this.loopPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureConsumerGroup(): Promise<void> {
|
||||
try {
|
||||
await this.garnetClient.createConsumerGroup(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!String(error).includes("BUSYGROUP")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runLoop(): Promise<void> {
|
||||
while (!this.stopRequested) {
|
||||
try {
|
||||
await this.runPeriodicChecksIfNeeded();
|
||||
|
||||
const recoveredPending = await this.recoverPendingOnce();
|
||||
if (recoveredPending) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const streams = await this.garnetClient.readGroup(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
this.consumerName,
|
||||
">",
|
||||
{
|
||||
blockMs: READ_BLOCK_MS,
|
||||
count: READ_COUNT,
|
||||
},
|
||||
);
|
||||
|
||||
if (!streams) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const stream of streams) {
|
||||
for (const message of stream.messages) {
|
||||
if (this.stopRequested) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.processMessage(
|
||||
message,
|
||||
message.deliveriesCounter ?? 1,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ err: error },
|
||||
"[ShadowWorker] Loop iteration failed",
|
||||
);
|
||||
await delay(1_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async recoverPendingOnce(): Promise<boolean> {
|
||||
const pending = await this.garnetClient.getPendingStreamRange(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
PENDING_BATCH_SIZE,
|
||||
);
|
||||
const recoverable = pending.filter(
|
||||
(entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS,
|
||||
);
|
||||
|
||||
if (recoverable.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const entry of recoverable) {
|
||||
if (this.stopRequested) {
|
||||
break;
|
||||
}
|
||||
|
||||
await this.recoverPendingEntry(entry);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async recoverPendingEntry(
|
||||
entry: StreamPendingEntry,
|
||||
): Promise<void> {
|
||||
const nextDeliveryCount = entry.deliveriesCounter + 1;
|
||||
const claimed = await this.garnetClient.claimPendingStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
this.consumerName,
|
||||
PENDING_IDLE_MS,
|
||||
entry.id,
|
||||
{
|
||||
retryCount: nextDeliveryCount,
|
||||
},
|
||||
);
|
||||
const message = claimed[0];
|
||||
|
||||
if (!message) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (nextDeliveryCount > MAX_DELIVERIES) {
|
||||
await this.handleMaxDeliveries(
|
||||
message,
|
||||
nextDeliveryCount,
|
||||
"recovered entry exceeded max deliveries",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.processMessage(message, nextDeliveryCount);
|
||||
}
|
||||
|
||||
private async processMessage(
|
||||
message: StreamEntry,
|
||||
deliveryCount: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
switch (getJobType(message) as BackgroundJobType) {
|
||||
case "shadow-eval":
|
||||
await this.processShadowEval(message);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
case "summarize":
|
||||
await this.processSummarize(message);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
default:
|
||||
this.logger.warn(
|
||||
{
|
||||
messageId: message.id,
|
||||
type: getJobType(message),
|
||||
},
|
||||
"[ShadowWorker] Unknown background job type; acknowledging",
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{
|
||||
err: error,
|
||||
deliveryCount,
|
||||
messageId: message.id,
|
||||
type: getJobType(message),
|
||||
},
|
||||
"[ShadowWorker] Job processing failed",
|
||||
);
|
||||
|
||||
if (deliveryCount >= MAX_DELIVERIES) {
|
||||
await this.handleMaxDeliveries(
|
||||
message,
|
||||
deliveryCount,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async processShadowEval(message: StreamEntry): Promise<void> {
|
||||
const job = parseShadowEvalJob(message);
|
||||
const validation = validateOnboardingResponse(
|
||||
job.response.text,
|
||||
job.derived,
|
||||
{
|
||||
allowShortResponse: job.derived.allowShortResponse,
|
||||
},
|
||||
);
|
||||
const llmJudge = await judgeOnboardingWithLlm(
|
||||
this.llmClient,
|
||||
toEvalCase(job, message.id),
|
||||
job.response.text,
|
||||
toDerivedSnapshot(job.derived),
|
||||
);
|
||||
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
||||
const llmJudgeErrored =
|
||||
llmJudge.skipped &&
|
||||
llmJudge.reason !== "empty-response" &&
|
||||
llmJudge.reason !== "disabled";
|
||||
|
||||
this.recordEvalObservation({
|
||||
llmJudgeErrored,
|
||||
llmOverall,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: job.latencyMs ?? null,
|
||||
});
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval-result",
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs:
|
||||
job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall:
|
||||
llmOverall === null ? "" : String(llmOverall),
|
||||
input: JSON.stringify({
|
||||
request: job.request,
|
||||
}),
|
||||
output: JSON.stringify(job.response),
|
||||
derived: JSON.stringify(job.derived),
|
||||
classified: JSON.stringify(job.classified),
|
||||
validation: JSON.stringify(validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
maxLen: getEvalStreamMaxLen(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async processSummarize(message: StreamEntry): Promise<void> {
|
||||
const chatroomId = message.message["chatroomId"];
|
||||
const userId = message.message["userId"];
|
||||
|
||||
if (!chatroomId?.trim() || !userId?.trim()) {
|
||||
throw new Error("Missing summarize job identifiers");
|
||||
}
|
||||
|
||||
const [history, existingSummary] = await Promise.all([
|
||||
this.conversationStore.getHistory(chatroomId, userId),
|
||||
this.conversationStore.getSummary(chatroomId, userId),
|
||||
]);
|
||||
|
||||
if (history.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const summary = await summarizeConversation(
|
||||
this.llmClient,
|
||||
history,
|
||||
existingSummary ?? undefined,
|
||||
);
|
||||
|
||||
await this.conversationStore.setSummary(chatroomId, userId, summary);
|
||||
}
|
||||
|
||||
private async handleMaxDeliveries(
|
||||
message: StreamEntry,
|
||||
deliveryCount: number,
|
||||
lastError: string,
|
||||
): Promise<void> {
|
||||
const type = getJobType(message);
|
||||
|
||||
if (type === "summarize") {
|
||||
this.logger.warn(
|
||||
{
|
||||
deliveryCount,
|
||||
lastError,
|
||||
messageId: message.id,
|
||||
},
|
||||
"[ShadowWorker] Dropping summarize job after max retries",
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
BACKGROUND_DLQ_STREAM_KEY,
|
||||
{
|
||||
type,
|
||||
payload: JSON.stringify(message.message),
|
||||
retryCount: String(deliveryCount),
|
||||
lastError: lastError.slice(0, 500),
|
||||
originalId: message.id,
|
||||
},
|
||||
{
|
||||
maxLen: 1_000,
|
||||
},
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
}
|
||||
|
||||
private acknowledge(id: string): Promise<number> {
|
||||
return this.garnetClient.acknowledgeStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
private recordEvalObservation(observation: EvalObservation): void {
|
||||
this.recentEvalObservations.push(observation);
|
||||
this.pruneOldObservations(Date.now());
|
||||
}
|
||||
|
||||
private pruneOldObservations(now: number): void {
|
||||
const cutoff = now - ALERT_WINDOW_MS;
|
||||
let removeCount = 0;
|
||||
|
||||
for (const observation of this.recentEvalObservations) {
|
||||
if (observation.recordedAt >= cutoff) {
|
||||
break;
|
||||
}
|
||||
removeCount += 1;
|
||||
}
|
||||
|
||||
if (removeCount > 0) {
|
||||
this.recentEvalObservations.splice(0, removeCount);
|
||||
}
|
||||
}
|
||||
|
||||
private async runPeriodicChecksIfNeeded(): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastAggregateAt = now;
|
||||
this.pruneOldObservations(now);
|
||||
|
||||
await this.checkLatencyAlert(now);
|
||||
await this.checkEvalScoreDropAlert(now);
|
||||
await this.checkLlmErrorRateAlert(now);
|
||||
await this.checkOpenCircuitAlerts(now);
|
||||
}
|
||||
|
||||
private async checkLatencyAlert(now: number): Promise<void> {
|
||||
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
|
||||
);
|
||||
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
||||
const threshold = getAlertThresholdP95LatencyMs();
|
||||
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: p95LatencyMs,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkEvalScoreDropAlert(now: number): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.llmOverall === null ? [] : [observation.llmOverall],
|
||||
);
|
||||
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentAverage =
|
||||
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
||||
const baselineAverage = await this.loadBaselineOverallScore();
|
||||
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
||||
const threshold = getAlertThresholdEvalScoreDropPct();
|
||||
|
||||
if (dropPct === null || dropPct <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "eval-score-drop",
|
||||
metric: "shadow_eval.llm_overall_drop_pct",
|
||||
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(dropPct.toFixed(3)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkLlmErrorRateAlert(now: number): Promise<void> {
|
||||
if (
|
||||
!this.alertSender ||
|
||||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
|
||||
const threshold = getAlertThresholdLlmErrorRate();
|
||||
if (errorRate === null || errorRate <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "llm-error-rate",
|
||||
metric: "shadow_eval.llm_error_rate",
|
||||
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(errorRate.toFixed(4)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkOpenCircuitAlerts(now: number): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*");
|
||||
|
||||
for (const key of circuitKeys) {
|
||||
const record = await this.garnetClient.getHash(key);
|
||||
if (record["state"] !== "open") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const endpointHash = key.split(":").at(-1) ?? "unknown";
|
||||
const failureCount = Number(record["failureCount"]);
|
||||
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "endpoint-down",
|
||||
metric: "llm.circuit.open",
|
||||
severity: "critical",
|
||||
suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.",
|
||||
threshold: 0,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number.isFinite(failureCount) ? failureCount : 1,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
},
|
||||
{
|
||||
dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadBaselineOverallScore(): Promise<number | null> {
|
||||
const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY);
|
||||
const redisParsed = parseBaselineOverallFromRaw(redisBaseline);
|
||||
if (redisParsed !== null) {
|
||||
return redisParsed;
|
||||
}
|
||||
|
||||
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const fileRaw = await readFile(filePath, "utf8");
|
||||
return parseBaselineOverallFromRaw(fileRaw);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{ err: error, filePath },
|
||||
"[ShadowWorker] Failed to read baseline file",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseBaselineOverallFromRaw(raw: string | null): number | null {
|
||||
if (!raw?.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as Partial<EvalBaselineFile>;
|
||||
const value = parsed.summary?.llmAverageOverall;
|
||||
return typeof value === "number" ? value : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
export interface EvalRoomContextMessage {
|
||||
senderId: string;
|
||||
nickname: string;
|
||||
message: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface EvalCaseInput {
|
||||
request: AgentRequest;
|
||||
conversationHistory?: ConversationMessage[] | undefined;
|
||||
roomContext?: EvalRoomContextMessage[] | undefined;
|
||||
session?: OnboardingSession | null | undefined;
|
||||
blackboard?: Blackboard | null | undefined;
|
||||
summary?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalCaseExpected {
|
||||
intent?: string | undefined;
|
||||
mustIncludeAny?: string[] | undefined;
|
||||
mustNotInclude?: string[] | undefined;
|
||||
responseMode?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalCaseDefinition {
|
||||
id: string;
|
||||
lineNumber: number;
|
||||
name?: string | undefined;
|
||||
notes?: string | undefined;
|
||||
input: EvalCaseInput;
|
||||
expected: EvalCaseExpected;
|
||||
}
|
||||
|
||||
export interface RuleJudgeCheck {
|
||||
name:
|
||||
| "validator"
|
||||
| "intent"
|
||||
| "responseMode"
|
||||
| "mustIncludeAny"
|
||||
| "mustNotInclude";
|
||||
passed: boolean;
|
||||
skipped: boolean;
|
||||
detail?: string | undefined;
|
||||
}
|
||||
|
||||
export interface RuleJudgeResult {
|
||||
passed: boolean;
|
||||
score: number;
|
||||
checks: RuleJudgeCheck[];
|
||||
validatorStatus: "pass" | "soft-warn" | "reject";
|
||||
validatorReasons: string[];
|
||||
}
|
||||
|
||||
export interface LlmJudgeScores {
|
||||
friendliness: number;
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
scope: number;
|
||||
overall: number;
|
||||
}
|
||||
|
||||
export type LlmJudgeResult =
|
||||
| {
|
||||
skipped: true;
|
||||
reason: string;
|
||||
raw?: string | undefined;
|
||||
}
|
||||
| {
|
||||
skipped: false;
|
||||
model: string;
|
||||
endpoint: string;
|
||||
provider: string;
|
||||
scores: LlmJudgeScores;
|
||||
rationale: string;
|
||||
raw: string;
|
||||
};
|
||||
|
||||
export interface EvalDerivedSnapshot {
|
||||
allowShortResponse: boolean;
|
||||
intent: string;
|
||||
phase: string;
|
||||
responseMode: DerivedState["responseMode"];
|
||||
shouldRespond: boolean;
|
||||
suggestedMaxTokens: number;
|
||||
tier: DerivedState["tier"];
|
||||
}
|
||||
|
||||
export interface EvalClassifiedSnapshot {
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: ClassifiedIntent["source"];
|
||||
}
|
||||
|
||||
export interface EvalCaseResult {
|
||||
id: string;
|
||||
lineNumber: number;
|
||||
name?: string | undefined;
|
||||
latencyMs: number;
|
||||
classified: EvalClassifiedSnapshot;
|
||||
derived: EvalDerivedSnapshot;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
ruleJudge: RuleJudgeResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
error?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalReportSummary {
|
||||
averageLatencyMs: number;
|
||||
cases: number;
|
||||
emptyResponses: number;
|
||||
llmAverageAccuracy: number | null;
|
||||
llmAverageConcision: number | null;
|
||||
llmAverageFriendliness: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
llmAverageScope: number | null;
|
||||
llmJudgedCases: number;
|
||||
passRate: number;
|
||||
passedCases: number;
|
||||
ruleAverageScore: number;
|
||||
}
|
||||
|
||||
export interface EvalSummaryDelta {
|
||||
averageLatencyMs: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
}
|
||||
|
||||
export interface EvalReport {
|
||||
baselineDelta?: EvalSummaryDelta | undefined;
|
||||
generatedAt: string;
|
||||
inputPath: string;
|
||||
setName: string;
|
||||
summary: EvalReportSummary;
|
||||
cases: EvalCaseResult[];
|
||||
}
|
||||
|
||||
export interface EvalBaselineFile {
|
||||
reportPath: string;
|
||||
setName: string;
|
||||
summary: EvalReportSummary;
|
||||
updatedAt: string;
|
||||
}
|
||||
Reference in new issue
Block a user