Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
+349
-28
@@ -13,22 +13,59 @@ import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { AlertSender } from "@obs/alert";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import {
|
||||
getAlertThresholdDriftPct,
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getEvalAggTickMs,
|
||||
getEvalArchiveTickMs,
|
||||
getEvalDriftTickMs,
|
||||
getEvalProcessLlmJudgeEnabled,
|
||||
getEvalProcessLlmJudgeSamplingRate,
|
||||
getEvalPromoteTickMs,
|
||||
getEvalReviewTickMs,
|
||||
getEvalStreamMaxLen,
|
||||
} from "../config.js";
|
||||
import {
|
||||
aggregateRecentEvalResults,
|
||||
EVAL_AGG_CURSOR_KEY,
|
||||
sliceKeyToString,
|
||||
} from "./aggregator.js";
|
||||
import {
|
||||
archiveEvalResults,
|
||||
EVAL_ARCHIVE_CURSOR_KEY,
|
||||
type MongoConnectionLike,
|
||||
} from "./archive.js";
|
||||
import { computeDriftReports } from "./drift.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import {
|
||||
judgeProcessRule,
|
||||
judgeProcessWithLlm,
|
||||
} from "./judges/process-judge.js";
|
||||
import {
|
||||
appendCandidatesToJsonl,
|
||||
selectPromotionCandidates,
|
||||
} from "./promote.js";
|
||||
import {
|
||||
EVAL_REVIEW_CURSOR_KEY,
|
||||
routeReviewCandidates,
|
||||
} from "./review/queue.js";
|
||||
import {
|
||||
archiveReviewLabels,
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
} from "./review/label-archive.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
ProcessJudgeResult,
|
||||
} from "./types.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
@@ -57,6 +94,7 @@ interface ShadowEvalJob {
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
process?: ProcessTraceSnapshot | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
@@ -112,6 +150,14 @@ function parseJsonField<T>(
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
@@ -140,8 +186,11 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
message.message["input"],
|
||||
"input",
|
||||
);
|
||||
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
|
||||
message.message["process"],
|
||||
);
|
||||
|
||||
return {
|
||||
const job: ShadowEvalJob = {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
message.message["classified"],
|
||||
"classified",
|
||||
@@ -158,6 +207,12 @@ function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
"output",
|
||||
),
|
||||
};
|
||||
|
||||
if (process) {
|
||||
job.process = process;
|
||||
}
|
||||
|
||||
return job;
|
||||
}
|
||||
|
||||
export function calculatePercentile(
|
||||
@@ -206,11 +261,18 @@ export function calculateLlmErrorRate(
|
||||
}
|
||||
|
||||
export class ShadowWorker {
|
||||
private lastAggregateAt = 0;
|
||||
private archiveFailureCount = 0;
|
||||
private lastAlertCheckAt = 0;
|
||||
private lastArchiveAt = 0;
|
||||
private lastDriftAt = 0;
|
||||
private lastEvalAggregateAt = 0;
|
||||
private lastPromoteAt = 0;
|
||||
private lastReviewAt = 0;
|
||||
private loopPromise: Promise<void> | null = null;
|
||||
private readonly logger = getLogger();
|
||||
private readonly recentEvalObservations: EvalObservation[] = [];
|
||||
private stopRequested = false;
|
||||
private warnedMissingMongo = false;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
@@ -218,6 +280,7 @@ export class ShadowWorker {
|
||||
private readonly conversationStore: ConversationStore,
|
||||
private readonly alertSender?: AlertSender,
|
||||
private readonly consumerName = `bg-worker-${process.pid}`,
|
||||
private readonly mongoConnection?: MongoConnectionLike | undefined,
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
@@ -432,6 +495,9 @@ export class ShadowWorker {
|
||||
llmJudge.skipped &&
|
||||
llmJudge.reason !== "empty-response" &&
|
||||
llmJudge.reason !== "disabled";
|
||||
const processJudge = job.process
|
||||
? await this.judgeProcess(job, message.id)
|
||||
: undefined;
|
||||
|
||||
this.recordEvalObservation({
|
||||
llmJudgeErrored,
|
||||
@@ -440,37 +506,81 @@ export class ShadowWorker {
|
||||
responseLatencyMs: job.latencyMs ?? null,
|
||||
});
|
||||
|
||||
const resultFields: Record<string, string> = {
|
||||
type: "shadow-eval-result",
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
phase: job.derived.phase.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall: llmOverall === null ? "" : String(llmOverall),
|
||||
input: JSON.stringify({
|
||||
request: job.request,
|
||||
}),
|
||||
output: JSON.stringify(job.response),
|
||||
derived: JSON.stringify(job.derived),
|
||||
classified: JSON.stringify(job.classified),
|
||||
validation: JSON.stringify(validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
if (job.process) {
|
||||
resultFields.process = JSON.stringify(job.process);
|
||||
}
|
||||
if (processJudge) {
|
||||
resultFields.processJudge = JSON.stringify(processJudge);
|
||||
}
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval-result",
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs:
|
||||
job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall:
|
||||
llmOverall === null ? "" : String(llmOverall),
|
||||
input: JSON.stringify({
|
||||
request: job.request,
|
||||
}),
|
||||
output: JSON.stringify(job.response),
|
||||
derived: JSON.stringify(job.derived),
|
||||
classified: JSON.stringify(job.classified),
|
||||
validation: JSON.stringify(validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
},
|
||||
resultFields,
|
||||
{
|
||||
maxLen: getEvalStreamMaxLen(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async judgeProcess(
|
||||
job: ShadowEvalJob,
|
||||
messageId: string,
|
||||
): Promise<ProcessJudgeResult> {
|
||||
if (!job.process) {
|
||||
throw new Error("Missing process trace");
|
||||
}
|
||||
|
||||
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
|
||||
if (
|
||||
!getEvalProcessLlmJudgeEnabled() ||
|
||||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
|
||||
) {
|
||||
return ruleResult;
|
||||
}
|
||||
|
||||
const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, {
|
||||
request: job.request,
|
||||
finalText: job.response.text,
|
||||
});
|
||||
|
||||
return {
|
||||
checks: [...ruleResult.checks, ...llmResult.checks],
|
||||
passedCount: ruleResult.passedCount + llmResult.passedCount,
|
||||
totalCount: ruleResult.totalCount + llmResult.totalCount,
|
||||
score: Number(
|
||||
(
|
||||
(ruleResult.score * ruleResult.totalCount +
|
||||
llmResult.score * llmResult.totalCount) /
|
||||
Math.max(1, ruleResult.totalCount + llmResult.totalCount)
|
||||
).toFixed(3),
|
||||
),
|
||||
rationale: llmResult.rationale,
|
||||
};
|
||||
}
|
||||
|
||||
private async processSummarize(message: StreamEntry): Promise<void> {
|
||||
const chatroomId = message.message["chatroomId"];
|
||||
const userId = message.message["userId"];
|
||||
@@ -563,16 +673,23 @@ export class ShadowWorker {
|
||||
}
|
||||
|
||||
private async runPeriodicChecksIfNeeded(): Promise<void> {
|
||||
const now = Date.now();
|
||||
|
||||
await this.runAggregationTickIfNeeded(now);
|
||||
await this.runArchiveTickIfNeeded(now);
|
||||
await this.runDriftTickIfNeeded(now);
|
||||
await this.runPromoteTickIfNeeded(now);
|
||||
await this.runReviewTickIfNeeded(now);
|
||||
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
|
||||
if (now - this.lastAlertCheckAt < ALERT_TICK_MS) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastAggregateAt = now;
|
||||
this.lastAlertCheckAt = now;
|
||||
this.pruneOldObservations(now);
|
||||
|
||||
await this.checkLatencyAlert(now);
|
||||
@@ -581,6 +698,210 @@ export class ShadowWorker {
|
||||
await this.checkOpenCircuitAlerts(now);
|
||||
}
|
||||
|
||||
private async runAggregationTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastEvalAggregateAt = now;
|
||||
const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY);
|
||||
const result = await aggregateRecentEvalResults(this.garnetClient, {
|
||||
sinceId: sinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
});
|
||||
|
||||
if (result.updatedSlices > 0) {
|
||||
this.logger.debug(
|
||||
{
|
||||
lastSeenId: result.lastSeenId,
|
||||
updatedSlices: result.updatedSlices,
|
||||
},
|
||||
"[ShadowWorker] Aggregated shadow eval results",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private warnMissingMongoOnce(): void {
|
||||
if (this.warnedMissingMongo) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.warnedMissingMongo = true;
|
||||
this.logger.warn(
|
||||
"[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled",
|
||||
);
|
||||
}
|
||||
|
||||
private async runArchiveTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastArchiveAt < getEvalArchiveTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastArchiveAt = now;
|
||||
if (!this.mongoConnection) {
|
||||
this.warnMissingMongoOnce();
|
||||
return;
|
||||
}
|
||||
|
||||
const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY);
|
||||
try {
|
||||
const result = await archiveEvalResults(
|
||||
this.garnetClient,
|
||||
this.mongoConnection,
|
||||
{
|
||||
sinceId: sinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
},
|
||||
);
|
||||
const labelSinceId = await this.garnetClient.getString(
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
);
|
||||
const labelResult = await archiveReviewLabels(
|
||||
this.garnetClient,
|
||||
this.mongoConnection,
|
||||
{
|
||||
sinceId: labelSinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
},
|
||||
);
|
||||
this.archiveFailureCount = 0;
|
||||
if (result.archivedCount > 0) {
|
||||
this.logger.info(
|
||||
{
|
||||
archivedCount: result.archivedCount,
|
||||
lastArchivedId: result.lastArchivedId,
|
||||
},
|
||||
"[ShadowWorker] Archived shadow eval results",
|
||||
);
|
||||
}
|
||||
if (labelResult.archivedCount > 0) {
|
||||
this.logger.info(
|
||||
{
|
||||
archivedCount: labelResult.archivedCount,
|
||||
lastArchivedId: labelResult.lastArchivedId,
|
||||
},
|
||||
"[ShadowWorker] Archived review labels",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.archiveFailureCount += 1;
|
||||
this.logger.warn(
|
||||
{
|
||||
err: error,
|
||||
failureCount: this.archiveFailureCount,
|
||||
},
|
||||
"[ShadowWorker] Failed to archive shadow eval results",
|
||||
);
|
||||
|
||||
if (this.archiveFailureCount >= 3 && this.alertSender) {
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "archive-stalled",
|
||||
metric: "shadow_eval.archive_failures",
|
||||
severity: "warn",
|
||||
suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.",
|
||||
threshold: 3,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: this.archiveFailureCount,
|
||||
window: "consecutive",
|
||||
},
|
||||
{
|
||||
dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled",
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runDriftTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastDriftAt < getEvalDriftTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastDriftAt = now;
|
||||
if (!this.mongoConnection) {
|
||||
this.warnMissingMongoOnce();
|
||||
return;
|
||||
}
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const threshold = getAlertThresholdDriftPct();
|
||||
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
|
||||
const significantReports = reports.filter((report) => report.significant);
|
||||
|
||||
for (const report of significantReports) {
|
||||
const slice = sliceKeyToString(report.slice);
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "eval-drift",
|
||||
metric: "shadow_eval.drift.llm_overall_drop_pct",
|
||||
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: report.dropPct,
|
||||
window: "24h_vs_7d",
|
||||
},
|
||||
{
|
||||
dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async runPromoteTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastPromoteAt < getEvalPromoteTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastPromoteAt = now;
|
||||
if (!this.mongoConnection) {
|
||||
this.warnMissingMongoOnce();
|
||||
return;
|
||||
}
|
||||
|
||||
const candidates = await selectPromotionCandidates(this.mongoConnection, {
|
||||
nowMs: now,
|
||||
});
|
||||
if (candidates.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await appendCandidatesToJsonl(candidates);
|
||||
this.logger.info(
|
||||
{
|
||||
appendedCount: result.appendedCount,
|
||||
outputPath: result.outputPath,
|
||||
},
|
||||
"[ShadowWorker] Promoted shadow eval candidates",
|
||||
);
|
||||
}
|
||||
|
||||
private async runReviewTickIfNeeded(now: number): Promise<void> {
|
||||
if (now - this.lastReviewAt < getEvalReviewTickMs()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastReviewAt = now;
|
||||
const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY);
|
||||
const result = await routeReviewCandidates(this.garnetClient, {
|
||||
sinceId: sinceId ?? "0-0",
|
||||
nowMs: now,
|
||||
});
|
||||
|
||||
if (result.enqueuedCount > 0) {
|
||||
this.logger.info(
|
||||
{
|
||||
enqueuedCount: result.enqueuedCount,
|
||||
lastSeenId: result.lastSeenId,
|
||||
},
|
||||
"[ShadowWorker] Routed eval results to review queue",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async checkLatencyAlert(now: number): Promise<void> {
|
||||
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
|
||||
|
||||
Reference in new issue
Block a user