Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit 6f09408423
57 files changed
+7060 -142

No files matched your search

+349 -28
View File
@@ -13,22 +13,59 @@ import type { LlmClient } from "@llm/llm-client";
import { getLogger } from "@obs/logger";
import type { AlertSender } from "@obs/alert";
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ConversationStore } from "@conv/conversation-store";
import {
getAlertThresholdDriftPct,
getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs,
getEvalAggTickMs,
getEvalArchiveTickMs,
getEvalDriftTickMs,
getEvalProcessLlmJudgeEnabled,
getEvalProcessLlmJudgeSamplingRate,
getEvalPromoteTickMs,
getEvalReviewTickMs,
getEvalStreamMaxLen,
} from "../config.js";
import {
aggregateRecentEvalResults,
EVAL_AGG_CURSOR_KEY,
sliceKeyToString,
} from "./aggregator.js";
import {
archiveEvalResults,
EVAL_ARCHIVE_CURSOR_KEY,
type MongoConnectionLike,
} from "./archive.js";
import { computeDriftReports } from "./drift.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import {
judgeProcessRule,
judgeProcessWithLlm,
} from "./judges/process-judge.js";
import {
appendCandidatesToJsonl,
selectPromotionCandidates,
} from "./promote.js";
import {
EVAL_REVIEW_CURSOR_KEY,
routeReviewCandidates,
} from "./review/queue.js";
import {
archiveReviewLabels,
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
} from "./review/label-archive.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalDerivedSnapshot,
ProcessJudgeResult,
} from "./types.js";
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
@@ -57,6 +94,7 @@ interface ShadowEvalJob {
createdAt: string;
derived: DerivedState;
latencyMs?: number | undefined;
process?: ProcessTraceSnapshot | undefined;
request: AgentRequest;
response: AgentResponse;
}
@@ -112,6 +150,14 @@ function parseJsonField<T>(
return JSON.parse(value) as T;
}
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
if (!value?.trim()) {
return undefined;
}
return JSON.parse(value) as T;
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
return {
allowShortResponse: derived.allowShortResponse,
@@ -140,8 +186,11 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
message.message["input"],
"input",
);
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
message.message["process"],
);
return {
const job: ShadowEvalJob = {
classified: parseJsonField<ClassifiedIntent>(
message.message["classified"],
"classified",
@@ -158,6 +207,12 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
"output",
),
};
if (process) {
job.process = process;
}
return job;
}
export function calculatePercentile(
@@ -206,11 +261,18 @@ export function calculateLlmErrorRate(
}
export class ShadowWorker {
private lastAggregateAt = 0;
private archiveFailureCount = 0;
private lastAlertCheckAt = 0;
private lastArchiveAt = 0;
private lastDriftAt = 0;
private lastEvalAggregateAt = 0;
private lastPromoteAt = 0;
private lastReviewAt = 0;
private loopPromise: Promise<void> | null = null;
private readonly logger = getLogger();
private readonly recentEvalObservations: EvalObservation[] = [];
private stopRequested = false;
private warnedMissingMongo = false;
constructor(
private readonly garnetClient: GarnetClient,
@@ -218,6 +280,7 @@ export class ShadowWorker {
private readonly conversationStore: ConversationStore,
private readonly alertSender?: AlertSender,
private readonly consumerName = `bg-worker-${process.pid}`,
private readonly mongoConnection?: MongoConnectionLike | undefined,
) {}
async start(): Promise<void> {
@@ -432,6 +495,9 @@ export class ShadowWorker {
llmJudge.skipped &&
llmJudge.reason !== "empty-response" &&
llmJudge.reason !== "disabled";
const processJudge = job.process
? await this.judgeProcess(job, message.id)
: undefined;
this.recordEvalObservation({
llmJudgeErrored,
@@ -440,37 +506,81 @@ export class ShadowWorker {
responseLatencyMs: job.latencyMs ?? null,
});
const resultFields: Record<string, string> = {
type: "shadow-eval-result",
requestId: job.request.requestId,
chatroomId: job.request.chatroomId,
userId: job.request.userId,
tier: String(job.derived.tier),
intent: job.derived.intent.kind,
phase: job.derived.phase.kind,
responseMode: job.derived.responseMode,
validatorStatus: validation.status,
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
llmOverall: llmOverall === null ? "" : String(llmOverall),
input: JSON.stringify({
request: job.request,
}),
output: JSON.stringify(job.response),
derived: JSON.stringify(job.derived),
classified: JSON.stringify(job.classified),
validation: JSON.stringify(validation),
llmJudge: JSON.stringify(llmJudge),
evaluatedAt: new Date().toISOString(),
};
if (job.process) {
resultFields.process = JSON.stringify(job.process);
}
if (processJudge) {
resultFields.processJudge = JSON.stringify(processJudge);
}
await this.garnetClient.appendStream(
EVAL_RESULTS_STREAM_KEY,
{
type: "shadow-eval-result",
requestId: job.request.requestId,
chatroomId: job.request.chatroomId,
userId: job.request.userId,
tier: String(job.derived.tier),
intent: job.derived.intent.kind,
responseMode: job.derived.responseMode,
validatorStatus: validation.status,
latencyMs:
job.latencyMs !== undefined ? String(job.latencyMs) : "",
llmOverall:
llmOverall === null ? "" : String(llmOverall),
input: JSON.stringify({
request: job.request,
}),
output: JSON.stringify(job.response),
derived: JSON.stringify(job.derived),
classified: JSON.stringify(job.classified),
validation: JSON.stringify(validation),
llmJudge: JSON.stringify(llmJudge),
evaluatedAt: new Date().toISOString(),
},
resultFields,
{
maxLen: getEvalStreamMaxLen(),
},
);
}
private async judgeProcess(
job: ShadowEvalJob,
messageId: string,
): Promise<ProcessJudgeResult> {
if (!job.process) {
throw new Error("Missing process trace");
}
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
if (
!getEvalProcessLlmJudgeEnabled() ||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
) {
return ruleResult;
}
const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, {
request: job.request,
finalText: job.response.text,
});
return {
checks: [...ruleResult.checks, ...llmResult.checks],
passedCount: ruleResult.passedCount + llmResult.passedCount,
totalCount: ruleResult.totalCount + llmResult.totalCount,
score: Number(
(
(ruleResult.score * ruleResult.totalCount +
llmResult.score * llmResult.totalCount) /
Math.max(1, ruleResult.totalCount + llmResult.totalCount)
).toFixed(3),
),
rationale: llmResult.rationale,
};
}
private async processSummarize(message: StreamEntry): Promise<void> {
const chatroomId = message.message["chatroomId"];
const userId = message.message["userId"];
@@ -563,16 +673,23 @@ export class ShadowWorker {
}
private async runPeriodicChecksIfNeeded(): Promise<void> {
const now = Date.now();
await this.runAggregationTickIfNeeded(now);
await this.runArchiveTickIfNeeded(now);
await this.runDriftTickIfNeeded(now);
await this.runPromoteTickIfNeeded(now);
await this.runReviewTickIfNeeded(now);
if (!this.alertSender) {
return;
}
const now = Date.now();
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
if (now - this.lastAlertCheckAt < ALERT_TICK_MS) {
return;
}
this.lastAggregateAt = now;
this.lastAlertCheckAt = now;
this.pruneOldObservations(now);
await this.checkLatencyAlert(now);
@@ -581,6 +698,210 @@ export class ShadowWorker {
await this.checkOpenCircuitAlerts(now);
}
private async runAggregationTickIfNeeded(now: number): Promise<void> {
if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) {
return;
}
this.lastEvalAggregateAt = now;
const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY);
const result = await aggregateRecentEvalResults(this.garnetClient, {
sinceId: sinceId ?? "0-0",
nowMs: now,
});
if (result.updatedSlices > 0) {
this.logger.debug(
{
lastSeenId: result.lastSeenId,
updatedSlices: result.updatedSlices,
},
"[ShadowWorker] Aggregated shadow eval results",
);
}
}
private warnMissingMongoOnce(): void {
if (this.warnedMissingMongo) {
return;
}
this.warnedMissingMongo = true;
this.logger.warn(
"[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled",
);
}
private async runArchiveTickIfNeeded(now: number): Promise<void> {
if (now - this.lastArchiveAt < getEvalArchiveTickMs()) {
return;
}
this.lastArchiveAt = now;
if (!this.mongoConnection) {
this.warnMissingMongoOnce();
return;
}
const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY);
try {
const result = await archiveEvalResults(
this.garnetClient,
this.mongoConnection,
{
sinceId: sinceId ?? "0-0",
nowMs: now,
},
);
const labelSinceId = await this.garnetClient.getString(
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
);
const labelResult = await archiveReviewLabels(
this.garnetClient,
this.mongoConnection,
{
sinceId: labelSinceId ?? "0-0",
nowMs: now,
},
);
this.archiveFailureCount = 0;
if (result.archivedCount > 0) {
this.logger.info(
{
archivedCount: result.archivedCount,
lastArchivedId: result.lastArchivedId,
},
"[ShadowWorker] Archived shadow eval results",
);
}
if (labelResult.archivedCount > 0) {
this.logger.info(
{
archivedCount: labelResult.archivedCount,
lastArchivedId: labelResult.lastArchivedId,
},
"[ShadowWorker] Archived review labels",
);
}
} catch (error) {
this.archiveFailureCount += 1;
this.logger.warn(
{
err: error,
failureCount: this.archiveFailureCount,
},
"[ShadowWorker] Failed to archive shadow eval results",
);
if (this.archiveFailureCount >= 3 && this.alertSender) {
await this.alertSender.sendAlert(
{
kind: "archive-stalled",
metric: "shadow_eval.archive_failures",
severity: "warn",
suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.",
threshold: 3,
timestamp: new Date(now).toISOString(),
value: this.archiveFailureCount,
window: "consecutive",
},
{
dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled",
},
);
}
}
}
private async runDriftTickIfNeeded(now: number): Promise<void> {
if (now - this.lastDriftAt < getEvalDriftTickMs()) {
return;
}
this.lastDriftAt = now;
if (!this.mongoConnection) {
this.warnMissingMongoOnce();
return;
}
if (!this.alertSender) {
return;
}
const threshold = getAlertThresholdDriftPct();
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
const significantReports = reports.filter((report) => report.significant);
for (const report of significantReports) {
const slice = sliceKeyToString(report.slice);
await this.alertSender.sendAlert(
{
kind: "eval-drift",
metric: "shadow_eval.drift.llm_overall_drop_pct",
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
threshold,
timestamp: new Date(now).toISOString(),
value: report.dropPct,
window: "24h_vs_7d",
},
{
dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`,
},
);
}
}
private async runPromoteTickIfNeeded(now: number): Promise<void> {
if (now - this.lastPromoteAt < getEvalPromoteTickMs()) {
return;
}
this.lastPromoteAt = now;
if (!this.mongoConnection) {
this.warnMissingMongoOnce();
return;
}
const candidates = await selectPromotionCandidates(this.mongoConnection, {
nowMs: now,
});
if (candidates.length === 0) {
return;
}
const result = await appendCandidatesToJsonl(candidates);
this.logger.info(
{
appendedCount: result.appendedCount,
outputPath: result.outputPath,
},
"[ShadowWorker] Promoted shadow eval candidates",
);
}
private async runReviewTickIfNeeded(now: number): Promise<void> {
if (now - this.lastReviewAt < getEvalReviewTickMs()) {
return;
}
this.lastReviewAt = now;
const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY);
const result = await routeReviewCandidates(this.garnetClient, {
sinceId: sinceId ?? "0-0",
nowMs: now,
});
if (result.enqueuedCount > 0) {
this.logger.info(
{
enqueuedCount: result.enqueuedCount,
lastSeenId: result.lastSeenId,
},
"[ShadowWorker] Routed eval results to review queue",
);
}
}
private async checkLatencyAlert(now: number): Promise<void> {
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],