import { existsSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import type { AgentRequest, AgentResponse } from "@agent/types"; import type { GarnetClient, StreamEntry, StreamPendingEntry, } from "@common/garnet"; import { summarizeConversation } from "@conv/summarizer"; import type { LlmClient } from "@llm/llm-client"; import { getLogger } from "@obs/logger"; import type { AlertSender } from "@obs/alert"; import type { ConversationStore } from "@conv/conversation-store"; import { getAlertThresholdDriftPct, getAlertThresholdEvalScoreDropPct, getAlertThresholdLlmErrorRate, getAlertThresholdP95LatencyMs, getEvalAggTickMs, getEvalArchiveTickMs, getEvalArchiveBatchSize, getEvalAttributionTickMs, getEvalDriftTickMs, getEvalProcessLlmJudgeEnabled, getEvalProcessLlmJudgeSamplingRate, getEvalPrmEnabled, getEvalPrmWeightsPath, getEvalPromoteTickMs, getEvalReviewTickMs, getEvalStreamMaxLen, } from "../config.js"; import { aggregateRecentEvalResults, EVAL_AGG_CURSOR_KEY, sliceKeyToString, } from "./aggregator.js"; import { archiveEvalResults, EVAL_ARCHIVE_CURSOR_KEY, getEvalArchiveCollection, type MongoConnectionLike, } from "./archive.js"; import { computeDriftReports } from "./drift.js"; import { judgeSkillWithLlm } from "./judges/llm-judge.js"; import { writeProcessAttributions } from "./judges/attribution.js"; import { loadPrmModel } from "./judges/prm.js"; import { judgeProcessRule, judgeProcessWithLlm, } from "./judges/process-judge.js"; import { getSkillJudgeConfig } from "./judges/skill-config.js"; import { appendCandidatesToJsonl, selectPromotionCandidates, } from "./promote.js"; import { EVAL_REVIEW_CURSOR_KEY, routeReviewCandidates, } from "./review/queue.js"; import { archiveReviewLabels, EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, } from "./review/label-archive.js"; import type { EvalBaselineFile, EvalCaseDefinition, EvalDerivedSnapshot, EvalProcessTraceSnapshot, ProcessJudgeResult, } from "./types.js"; const BACKGROUND_STREAM_KEY = "lolympus-agent:background"; const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq"; const BACKGROUND_GROUP = "bg-worker"; const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline"; const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results"; const EVAL_BASELINE_FILES = [ "eval-reports/baseline.onboarding.json", "eval-reports/baseline.json", ]; const ALERT_WINDOW_MS = 5 * 60_000; const ALERT_WINDOW_LABEL = "5m"; const ALERT_TICK_MS = 60_000; const MAX_DELIVERIES = 3; const MIN_ERROR_RATE_OBSERVATIONS = 5; const MIN_LATENCY_OBSERVATIONS = 5; const MIN_SCORE_OBSERVATIONS = 3; const PENDING_IDLE_MS = 1_000; const READ_BLOCK_MS = 5_000; const READ_COUNT = 10; const PENDING_BATCH_SIZE = 20; const STOP_TIMEOUT_MS = 10_000; type BackgroundJobType = "shadow-eval" | "summarize"; interface ShadowEvalJob { classified: unknown; createdAt: string; derived: unknown; latencyMs?: number | undefined; process?: EvalProcessTraceSnapshot | undefined; request: AgentRequest; response: AgentResponse; skill: string; version: number; } interface EvalObservation { llmJudgeErrored: boolean; llmOverall: number | null; recordedAt: number; responseLatencyMs: number | null; skill: string; } function delay(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, ms); }); } function promiseWithTimeout( promise: Promise, timeoutMs: number, ): Promise { return Promise.race([ promise, new Promise((_, reject) => { setTimeout(() => { reject(new Error(`Timed out after ${timeoutMs}ms`)); }, timeoutMs); }), ]); } function getJobType(message: StreamEntry): string { return message.message["type"] ?? "unknown"; } function parseOptionalNumberField(value: string | undefined): number | undefined { if (!value?.trim()) { return undefined; } const parsed = Number(value); return Number.isFinite(parsed) ? parsed : undefined; } function parseJsonField( value: string | undefined, fieldName: string, ): T { if (!value?.trim()) { throw new Error(`Missing ${fieldName} field`); } return JSON.parse(value) as T; } function parseOptionalJsonField(value: string | undefined): T | undefined { if (!value?.trim()) { return undefined; } return JSON.parse(value) as T; } function asRecord(value: unknown): Record { return typeof value === "object" && value !== null ? (value as Record) : {}; } function getStringField( record: Record, key: string, ): string | null { const value = record[key]; return typeof value === "string" && value.trim() ? value : null; } function getNestedString( record: Record, first: string, second: string, ): string | null { return getStringField(asRecord(record[first]), second); } function parseTier(value: unknown): 0 | 1 | 2 { const parsed = typeof value === "number" ? value : Number(value); return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0; } function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] { return value === "skip" || value === "template" || value === "llm" || value === "llm-with-tools" ? value : "llm"; } function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot { const record = asRecord(derived); const intent = getStringField(record, "intent") ?? getNestedString(record, "intent", "kind") ?? "unknown"; const phase = getStringField(record, "phase") ?? getNestedString(record, "phase", "kind") ?? "unknown"; const abstentionScore = record["abstentionScore"]; const suggestedMaxTokens = record["suggestedMaxTokens"]; return { ...(typeof abstentionScore === "number" ? { abstentionScore } : {}), allowShortResponse: record["allowShortResponse"] === true, intent, phase, responseMode: parseResponseMode(record["responseMode"]), ...(record["routingDecision"] ? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] } : {}), shouldRespond: record["shouldRespond"] !== false, suggestedMaxTokens: typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens) ? suggestedMaxTokens : 192, tier: parseTier(record["tier"]), }; } function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined): | NonNullable["output"]["abstention"] | undefined { return process?.steps.classify?.output.abstention; } function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition { return { id: `shadow:${messageId}`, lineNumber: 0, input: { request: job.request, }, expected: {}, }; } function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob { const input = parseJsonField<{ request: AgentRequest }>( message.message["input"], "input", ); const process = parseOptionalJsonField( message.message["process"], ); const job: ShadowEvalJob = { classified: parseJsonField( message.message["classified"], "classified", ), createdAt: message.message["createdAt"] ?? new Date().toISOString(), derived: parseJsonField( message.message["derived"], "derived", ), latencyMs: parseOptionalNumberField(message.message["latencyMs"]), request: input.request, response: parseJsonField( message.message["output"], "output", ), skill: message.message["skill"]?.trim() || input.request.skill || "onboarding", version: Number(message.message["version"] ?? 1), }; if (process) { job.process = process; } return job; } export function calculatePercentile( values: number[], percentile: number, ): number | null { if (values.length === 0) { return null; } const sorted = [...values].sort((left, right) => left - right); const index = Math.min( sorted.length - 1, Math.max(0, Math.ceil(sorted.length * percentile) - 1), ); const selected = sorted[index]; return selected === undefined ? null : selected; } export function calculateEvalScoreDropPct( currentAverage: number | null, baselineAverage: number | null, ): number | null { if ( currentAverage === null || baselineAverage === null || baselineAverage <= 0 ) { return null; } return ((baselineAverage - currentAverage) / baselineAverage) * 100; } export function calculateLlmErrorRate( observations: EvalObservation[], ): number | null { if (observations.length === 0) { return null; } const errorCount = observations.filter( (observation) => observation.llmJudgeErrored, ).length; return errorCount / observations.length; } function groupObservationsBySkill( observations: EvalObservation[], ): Map { const groups = new Map(); for (const observation of observations) { const values = groups.get(observation.skill) ?? []; values.push(observation); groups.set(observation.skill, values); } return groups; } export class ShadowWorker { private archiveFailureCount = 0; private lastAlertCheckAt = 0; private lastArchiveAt = 0; private lastAttributionAt = 0; private lastDriftAt = 0; private lastEvalAggregateAt = 0; private lastPromoteAt = 0; private lastReviewAt = 0; private loopPromise: Promise | null = null; private readonly logger = getLogger(); private readonly recentEvalObservations: EvalObservation[] = []; private stopRequested = false; private warnedMissingMongo = false; constructor( private readonly garnetClient: GarnetClient, private readonly llmClient: LlmClient, private readonly conversationStore: ConversationStore, private readonly alertSender?: AlertSender, private readonly consumerName = `bg-worker-${process.pid}`, private readonly mongoConnection?: MongoConnectionLike | undefined, ) {} async start(): Promise { await this.ensureConsumerGroup(); if (this.loopPromise) { return; } this.stopRequested = false; this.loopPromise = this.runLoop(); } async stop(): Promise { this.stopRequested = true; if (!this.loopPromise) { return; } try { await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS); } catch (error) { this.logger.warn( { err: error }, "[ShadowWorker] Stop timed out while waiting for the loop to finish", ); } finally { this.loopPromise = null; } } private async ensureConsumerGroup(): Promise { try { await this.garnetClient.createConsumerGroup( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, ); } catch (error) { if (!String(error).includes("BUSYGROUP")) { throw error; } } } private async runLoop(): Promise { while (!this.stopRequested) { try { await this.runPeriodicChecksIfNeeded(); const recoveredPending = await this.recoverPendingOnce(); if (recoveredPending) { continue; } const streams = await this.garnetClient.readGroup( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, this.consumerName, ">", { blockMs: READ_BLOCK_MS, count: READ_COUNT, }, ); if (!streams) { continue; } for (const stream of streams) { for (const message of stream.messages) { if (this.stopRequested) { return; } await this.processMessage( message, message.deliveriesCounter ?? 1, ); } } } catch (error) { this.logger.error( { err: error }, "[ShadowWorker] Loop iteration failed", ); await delay(1_000); } } } private async recoverPendingOnce(): Promise { const pending = await this.garnetClient.getPendingStreamRange( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, PENDING_BATCH_SIZE, ); const recoverable = pending.filter( (entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS, ); if (recoverable.length === 0) { return false; } for (const entry of recoverable) { if (this.stopRequested) { break; } await this.recoverPendingEntry(entry); } return true; } private async recoverPendingEntry( entry: StreamPendingEntry, ): Promise { const nextDeliveryCount = entry.deliveriesCounter + 1; const claimed = await this.garnetClient.claimPendingStream( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, this.consumerName, PENDING_IDLE_MS, entry.id, { retryCount: nextDeliveryCount, }, ); const message = claimed[0]; if (!message) { return; } if (nextDeliveryCount > MAX_DELIVERIES) { await this.handleMaxDeliveries( message, nextDeliveryCount, "recovered entry exceeded max deliveries", ); return; } await this.processMessage(message, nextDeliveryCount); } private async processMessage( message: StreamEntry, deliveryCount: number, ): Promise { try { switch (getJobType(message) as BackgroundJobType) { case "shadow-eval": await this.processShadowEval(message); await this.acknowledge(message.id); return; case "summarize": await this.processSummarize(message); await this.acknowledge(message.id); return; default: this.logger.warn( { messageId: message.id, type: getJobType(message), }, "[ShadowWorker] Unknown background job type; acknowledging", ); await this.acknowledge(message.id); } } catch (error) { this.logger.warn( { err: error, deliveryCount, messageId: message.id, type: getJobType(message), }, "[ShadowWorker] Job processing failed", ); if (deliveryCount >= MAX_DELIVERIES) { await this.handleMaxDeliveries( message, deliveryCount, error instanceof Error ? error.message : String(error), ); } } } private async processShadowEval(message: StreamEntry): Promise { const job = parseShadowEvalJob(message); const judgeConfig = getSkillJudgeConfig(job.skill); const derivedSnapshot = toDerivedSnapshot(job.derived); const abstention = getTraceAbstention(job.process); const responseTextForValidation = job.response.text.trim() ? job.response.text : abstention?.action.kind === "no-response" ? "[NO_RESPONSE]" : job.response.text; const validation = judgeConfig.validator ? judgeConfig.validator(responseTextForValidation, job.derived) : { reasons: [], status: "pass" as const, text: responseTextForValidation, }; const llmJudge = await judgeSkillWithLlm( this.llmClient, job.skill, toEvalCase(job, message.id), job.response.text, derivedSnapshot, ); const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall; const llmJudgeErrored = llmJudge.skipped && llmJudge.reason !== "empty-response" && llmJudge.reason !== "disabled"; const processJudge = job.process ? await this.judgeProcess(job, message.id) : undefined; this.recordEvalObservation({ llmJudgeErrored, llmOverall, recordedAt: Date.now(), responseLatencyMs: job.latencyMs ?? null, skill: job.skill, }); const resultFields: Record = { type: "shadow-eval-result", requestId: job.request.requestId, chatroomId: job.request.chatroomId, userId: job.request.userId, skill: job.skill, version: String(Math.max(2, job.version)), tier: String(derivedSnapshot.tier), intent: derivedSnapshot.intent, phase: derivedSnapshot.phase, responseMode: derivedSnapshot.responseMode, validatorStatus: validation.status, latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "", llmOverall: llmOverall === null ? "" : String(llmOverall), input: JSON.stringify({ request: job.request, }), output: JSON.stringify(job.response), derived: JSON.stringify(job.derived), classified: JSON.stringify(job.classified), validation: JSON.stringify(validation), llmJudge: JSON.stringify(llmJudge), evaluatedAt: new Date().toISOString(), }; if (abstention) { resultFields.abstentionAction = abstention.action.kind; resultFields.abstentionReason = abstention.reason ?? ""; resultFields.abstentionScore = String(abstention.score); } if (job.process) { resultFields.process = JSON.stringify(job.process); } if (processJudge) { resultFields.processJudge = JSON.stringify(processJudge); } await this.garnetClient.appendStream( EVAL_RESULTS_STREAM_KEY, resultFields, { maxLen: getEvalStreamMaxLen(), }, ); } private async judgeProcess( job: ShadowEvalJob, messageId: string, ): Promise { if (!job.process) { throw new Error("Missing process trace"); } const judgeConfig = getSkillJudgeConfig(job.skill); const ruleResult = judgeProcessRule( toEvalCase(job, messageId).expected, job.process, judgeConfig.toolRegistryFactory(), ); if ( !getEvalProcessLlmJudgeEnabled() || Math.random() >= getEvalProcessLlmJudgeSamplingRate() ) { return ruleResult; } const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, { request: job.request, finalText: job.response.text, }); const merged: ProcessJudgeResult = { checks: [...ruleResult.checks, ...llmResult.checks], passedCount: ruleResult.passedCount + llmResult.passedCount, totalCount: ruleResult.totalCount + llmResult.totalCount, score: Number( ( (ruleResult.score * ruleResult.totalCount + llmResult.score * llmResult.totalCount) / Math.max(1, ruleResult.totalCount + llmResult.totalCount) ).toFixed(3), ), rationale: llmResult.rationale, }; const prmSource = llmResult.prmFeatures ? llmResult : ruleResult; if (prmSource.prmFeatures) { merged.prmFeatures = prmSource.prmFeatures; } if (prmSource.prmScore !== undefined) { merged.prmScore = prmSource.prmScore; } if (prmSource.rootCauseStep) { merged.rootCauseStep = prmSource.rootCauseStep; } if (prmSource.rootCauseScoreLoss !== undefined) { merged.rootCauseScoreLoss = prmSource.rootCauseScoreLoss; } return merged; } private async processSummarize(message: StreamEntry): Promise { const chatroomId = message.message["chatroomId"]; const userId = message.message["userId"]; if (!chatroomId?.trim() || !userId?.trim()) { throw new Error("Missing summarize job identifiers"); } const [history, existingSummary] = await Promise.all([ this.conversationStore.getHistory(chatroomId, userId), this.conversationStore.getSummary(chatroomId, userId), ]); if (history.length === 0) { return; } const summary = await summarizeConversation( this.llmClient, history, existingSummary ?? undefined, ); await this.conversationStore.setSummary(chatroomId, userId, summary); } private async handleMaxDeliveries( message: StreamEntry, deliveryCount: number, lastError: string, ): Promise { const type = getJobType(message); if (type === "summarize") { this.logger.warn( { deliveryCount, lastError, messageId: message.id, }, "[ShadowWorker] Dropping summarize job after max retries", ); await this.acknowledge(message.id); return; } await this.garnetClient.appendStream( BACKGROUND_DLQ_STREAM_KEY, { type, payload: JSON.stringify(message.message), retryCount: String(deliveryCount), lastError: lastError.slice(0, 500), originalId: message.id, }, { maxLen: 1_000, }, ); await this.acknowledge(message.id); } private acknowledge(id: string): Promise { return this.garnetClient.acknowledgeStream( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, id, ); } private recordEvalObservation(observation: EvalObservation): void { this.recentEvalObservations.push(observation); this.pruneOldObservations(Date.now()); } private pruneOldObservations(now: number): void { const cutoff = now - ALERT_WINDOW_MS; let removeCount = 0; for (const observation of this.recentEvalObservations) { if (observation.recordedAt >= cutoff) { break; } removeCount += 1; } if (removeCount > 0) { this.recentEvalObservations.splice(0, removeCount); } } private async runPeriodicChecksIfNeeded(): Promise { const now = Date.now(); await this.runAggregationTickIfNeeded(now); await this.runArchiveTickIfNeeded(now); await this.runAttributionTickIfNeeded(now); await this.runDriftTickIfNeeded(now); await this.runPromoteTickIfNeeded(now); await this.runReviewTickIfNeeded(now); if (!this.alertSender) { return; } if (now - this.lastAlertCheckAt < ALERT_TICK_MS) { return; } this.lastAlertCheckAt = now; this.pruneOldObservations(now); await this.checkLatencyAlert(now); await this.checkEvalScoreDropAlert(now); await this.checkLlmErrorRateAlert(now); await this.checkOpenCircuitAlerts(now); } private async runAggregationTickIfNeeded(now: number): Promise { if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) { return; } this.lastEvalAggregateAt = now; const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY); const result = await aggregateRecentEvalResults(this.garnetClient, { sinceId: sinceId ?? "0-0", nowMs: now, }); if (result.updatedSlices > 0) { this.logger.debug( { lastSeenId: result.lastSeenId, updatedSlices: result.updatedSlices, }, "[ShadowWorker] Aggregated shadow eval results", ); } } private warnMissingMongoOnce(): void { if (this.warnedMissingMongo) { return; } this.warnedMissingMongo = true; this.logger.warn( "[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled", ); } private async runArchiveTickIfNeeded(now: number): Promise { if (now - this.lastArchiveAt < getEvalArchiveTickMs()) { return; } this.lastArchiveAt = now; if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY); try { const result = await archiveEvalResults( this.garnetClient, this.mongoConnection, { sinceId: sinceId ?? "0-0", nowMs: now, }, ); const labelSinceId = await this.garnetClient.getString( EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, ); const labelResult = await archiveReviewLabels( this.garnetClient, this.mongoConnection, { sinceId: labelSinceId ?? "0-0", nowMs: now, }, ); this.archiveFailureCount = 0; if (result.archivedCount > 0) { this.logger.info( { archivedCount: result.archivedCount, lastArchivedId: result.lastArchivedId, }, "[ShadowWorker] Archived shadow eval results", ); } if (labelResult.archivedCount > 0) { this.logger.info( { archivedCount: labelResult.archivedCount, lastArchivedId: labelResult.lastArchivedId, }, "[ShadowWorker] Archived review labels", ); } } catch (error) { this.archiveFailureCount += 1; this.logger.warn( { err: error, failureCount: this.archiveFailureCount, }, "[ShadowWorker] Failed to archive shadow eval results", ); if (this.archiveFailureCount >= 3 && this.alertSender) { await this.alertSender.sendAlert( { kind: "archive-stalled", metric: "shadow_eval.archive_failures", severity: "warn", suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.", threshold: 3, timestamp: new Date(now).toISOString(), value: this.archiveFailureCount, window: "consecutive", }, { dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled", }, ); } } } private async runDriftTickIfNeeded(now: number): Promise { if (now - this.lastDriftAt < getEvalDriftTickMs()) { return; } this.lastDriftAt = now; if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } if (!this.alertSender) { return; } const reports = await computeDriftReports(this.mongoConnection, { nowMs: now }); const significantReports = reports.filter((report) => { const threshold = getAlertThresholdDriftPct(report.slice.skill); return report.recent24hCount >= 30 && report.dropPct >= threshold; }); for (const report of significantReports) { const slice = sliceKeyToString(report.slice); const skill = report.slice.skill ?? "onboarding"; const threshold = getAlertThresholdDriftPct(skill); await this.alertSender.sendAlert( { kind: "eval-drift", metric: "shadow_eval.drift.llm_overall_drop_pct", severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn", skill, suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`, threshold, timestamp: new Date(now).toISOString(), value: report.dropPct, window: "24h_vs_7d", }, { dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`, }, ); } } private async runAttributionTickIfNeeded(now: number): Promise { if (now - this.lastAttributionAt < getEvalAttributionTickMs()) { return; } this.lastAttributionAt = now; if (!getEvalPrmEnabled()) { return; } if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } const model = loadPrmModel(getEvalPrmWeightsPath()); if (!model) { return; } const entries = await getEvalArchiveCollection(this.mongoConnection) .find({ process: { $exists: true }, }) .sort({ evaluatedAt: -1 }) .limit(getEvalArchiveBatchSize()) .toArray(); const result = await writeProcessAttributions( this.mongoConnection, entries, model, { nowMs: now }, ); if (result.attributedCount > 0) { this.logger.info( { attributedCount: result.attributedCount, modifiedCount: result.modifiedCount, upsertedCount: result.upsertedCount, }, "[ShadowWorker] Wrote process attribution entries", ); } } private async runPromoteTickIfNeeded(now: number): Promise { if (now - this.lastPromoteAt < getEvalPromoteTickMs()) { return; } this.lastPromoteAt = now; if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } const candidates = await selectPromotionCandidates(this.mongoConnection, { nowMs: now, }); if (candidates.length === 0) { return; } const result = await appendCandidatesToJsonl(candidates); this.logger.info( { appendedCount: result.appendedCount, outputPath: result.outputPath, }, "[ShadowWorker] Promoted shadow eval candidates", ); } private async runReviewTickIfNeeded(now: number): Promise { if (now - this.lastReviewAt < getEvalReviewTickMs()) { return; } this.lastReviewAt = now; const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY); const result = await routeReviewCandidates(this.garnetClient, { sinceId: sinceId ?? "0-0", nowMs: now, }); if (result.enqueuedCount > 0) { this.logger.info( { enqueuedCount: result.enqueuedCount, lastSeenId: result.lastSeenId, }, "[ShadowWorker] Routed eval results to review queue", ); } } private async checkLatencyAlert(now: number): Promise { if (!this.alertSender) { return; } for (const [skill, observations] of groupObservationsBySkill( this.recentEvalObservations, )) { const latencySamples = observations.flatMap((observation) => observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs], ); if (latencySamples.length < MIN_LATENCY_OBSERVATIONS) { continue; } const p95LatencyMs = calculatePercentile(latencySamples, 0.95); const threshold = getAlertThresholdP95LatencyMs(skill); if (p95LatencyMs === null || p95LatencyMs <= threshold) { continue; } await this.alertSender.sendAlert({ kind: "latency", metric: "shadow_eval.p95_latency_ms", severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn", skill, suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.", threshold, timestamp: new Date(now).toISOString(), value: p95LatencyMs, window: ALERT_WINDOW_LABEL, }); } } private async checkEvalScoreDropAlert(now: number): Promise { if (!this.alertSender) { return; } const baselineAverage = await this.loadBaselineOverallScore(); for (const [skill, observations] of groupObservationsBySkill( this.recentEvalObservations, )) { const scoreSamples = observations.flatMap((observation) => observation.llmOverall === null ? [] : [observation.llmOverall], ); if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) { continue; } const currentAverage = scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length; const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage); const threshold = getAlertThresholdEvalScoreDropPct(skill); if (dropPct === null || dropPct <= threshold) { continue; } await this.alertSender.sendAlert({ kind: "eval-score-drop", metric: "shadow_eval.llm_overall_drop_pct", severity: dropPct >= threshold * 1.5 ? "critical" : "warn", skill, suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.", threshold, timestamp: new Date(now).toISOString(), value: Number(dropPct.toFixed(3)), window: ALERT_WINDOW_LABEL, }); } } private async checkLlmErrorRateAlert(now: number): Promise { if (!this.alertSender) { return; } for (const [skill, observations] of groupObservationsBySkill( this.recentEvalObservations, )) { if (observations.length < MIN_ERROR_RATE_OBSERVATIONS) { continue; } const errorRate = calculateLlmErrorRate(observations); const threshold = getAlertThresholdLlmErrorRate(skill); if (errorRate === null || errorRate <= threshold) { continue; } await this.alertSender.sendAlert({ kind: "llm-error-rate", metric: "shadow_eval.llm_error_rate", severity: errorRate >= threshold * 1.5 ? "critical" : "warn", skill, suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.", threshold, timestamp: new Date(now).toISOString(), value: Number(errorRate.toFixed(4)), window: ALERT_WINDOW_LABEL, }); } } private async checkOpenCircuitAlerts(now: number): Promise { if (!this.alertSender) { return; } const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*"); for (const key of circuitKeys) { const record = await this.garnetClient.getHash(key); if (record["state"] !== "open") { continue; } const endpointHash = key.split(":").at(-1) ?? "unknown"; const failureCount = Number(record["failureCount"]); await this.alertSender.sendAlert( { kind: "endpoint-down", metric: "llm.circuit.open", severity: "critical", suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.", threshold: 0, timestamp: new Date(now).toISOString(), value: Number.isFinite(failureCount) ? failureCount : 1, window: ALERT_WINDOW_LABEL, }, { dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`, }, ); } } private async loadBaselineOverallScore(): Promise { const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY); const redisParsed = parseBaselineOverallFromRaw(redisBaseline); if (redisParsed !== null) { return redisParsed; } const filePath = EVAL_BASELINE_FILES .map((candidate) => resolve(process.cwd(), candidate)) .find((candidate) => existsSync(candidate)); if (!filePath) { return null; } try { const fileRaw = await readFile(filePath, "utf8"); return parseBaselineOverallFromRaw(fileRaw); } catch (error) { this.logger.warn( { err: error, filePath }, "[ShadowWorker] Failed to read baseline file", ); return null; } } } function parseBaselineOverallFromRaw(raw: string | null): number | null { if (!raw?.trim()) { return null; } try { const parsed = JSON.parse(raw) as Partial; const value = parsed.summary?.llmAverageOverall; return typeof value === "number" ? value : null; } catch { return null; } }