import { existsSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import type { AgentRequest, AgentResponse } from "@agent/types"; import type { GarnetClient, StreamEntry, StreamPendingEntry, } from "@common/garnet"; import { summarizeConversation } from "@conv/summarizer"; import type { LlmClient } from "@llm/llm-client"; import { getLogger } from "@obs/logger"; import type { AlertSender } from "@obs/alert"; import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator"; import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import type { ClassifiedIntent, DerivedState, } from "@skills/onboarding/state/onboarding-state"; import type { ConversationStore } from "@conv/conversation-store"; import { getAlertThresholdDriftPct, getAlertThresholdEvalScoreDropPct, getAlertThresholdLlmErrorRate, getAlertThresholdP95LatencyMs, getEvalAggTickMs, getEvalArchiveTickMs, getEvalArchiveBatchSize, getEvalAttributionTickMs, getEvalDriftTickMs, getEvalProcessLlmJudgeEnabled, getEvalProcessLlmJudgeSamplingRate, getEvalPrmEnabled, getEvalPrmWeightsPath, getEvalPromoteTickMs, getEvalReviewTickMs, getEvalStreamMaxLen, } from "../config.js"; import { aggregateRecentEvalResults, EVAL_AGG_CURSOR_KEY, sliceKeyToString, } from "./aggregator.js"; import { archiveEvalResults, EVAL_ARCHIVE_CURSOR_KEY, getEvalArchiveCollection, type MongoConnectionLike, } from "./archive.js"; import { computeDriftReports } from "./drift.js"; import { judgeOnboardingWithLlm } from "./judges/llm-judge.js"; import { writeProcessAttributions } from "./judges/attribution.js"; import { loadPrmModel } from "./judges/prm.js"; import { judgeProcessRule, judgeProcessWithLlm, } from "./judges/process-judge.js"; import { appendCandidatesToJsonl, selectPromotionCandidates, } from "./promote.js"; import { EVAL_REVIEW_CURSOR_KEY, routeReviewCandidates, } from "./review/queue.js"; import { archiveReviewLabels, EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, } from "./review/label-archive.js"; import type { EvalBaselineFile, EvalCaseDefinition, EvalDerivedSnapshot, ProcessJudgeResult, } from "./types.js"; const BACKGROUND_STREAM_KEY = "lolympus-agent:background"; const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq"; const BACKGROUND_GROUP = "bg-worker"; const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline"; const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results"; const EVAL_BASELINE_FILE = "eval-reports/baseline.json"; const ALERT_WINDOW_MS = 5 * 60_000; const ALERT_WINDOW_LABEL = "5m"; const ALERT_TICK_MS = 60_000; const MAX_DELIVERIES = 3; const MIN_ERROR_RATE_OBSERVATIONS = 5; const MIN_LATENCY_OBSERVATIONS = 5; const MIN_SCORE_OBSERVATIONS = 3; const PENDING_IDLE_MS = 1_000; const READ_BLOCK_MS = 5_000; const READ_COUNT = 10; const PENDING_BATCH_SIZE = 20; const STOP_TIMEOUT_MS = 10_000; type BackgroundJobType = "shadow-eval" | "summarize"; interface ShadowEvalJob { classified: ClassifiedIntent; createdAt: string; derived: DerivedState; latencyMs?: number | undefined; process?: ProcessTraceSnapshot | undefined; request: AgentRequest; response: AgentResponse; } interface EvalObservation { llmJudgeErrored: boolean; llmOverall: number | null; recordedAt: number; responseLatencyMs: number | null; } function delay(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, ms); }); } function promiseWithTimeout( promise: Promise, timeoutMs: number, ): Promise { return Promise.race([ promise, new Promise((_, reject) => { setTimeout(() => { reject(new Error(`Timed out after ${timeoutMs}ms`)); }, timeoutMs); }), ]); } function getJobType(message: StreamEntry): string { return message.message["type"] ?? "unknown"; } function parseOptionalNumberField(value: string | undefined): number | undefined { if (!value?.trim()) { return undefined; } const parsed = Number(value); return Number.isFinite(parsed) ? parsed : undefined; } function parseJsonField( value: string | undefined, fieldName: string, ): T { if (!value?.trim()) { throw new Error(`Missing ${fieldName} field`); } return JSON.parse(value) as T; } function parseOptionalJsonField(value: string | undefined): T | undefined { if (!value?.trim()) { return undefined; } return JSON.parse(value) as T; } function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot { return { ...(derived.abstentionScore !== undefined ? { abstentionScore: derived.abstentionScore } : {}), allowShortResponse: derived.allowShortResponse, intent: derived.intent.kind, phase: derived.phase.kind, responseMode: derived.responseMode, ...(derived.routingDecision ? { routingDecision: derived.routingDecision } : {}), shouldRespond: derived.shouldRespond, suggestedMaxTokens: derived.suggestedMaxTokens, tier: derived.tier, }; } function getTraceAbstention(process: ProcessTraceSnapshot | undefined): | NonNullable["output"]["abstention"] | undefined { return process?.steps.classify?.output.abstention; } function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition { return { id: `shadow:${messageId}`, lineNumber: 0, input: { request: job.request, }, expected: {}, }; } function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob { const input = parseJsonField<{ request: AgentRequest }>( message.message["input"], "input", ); const process = parseOptionalJsonField( message.message["process"], ); const job: ShadowEvalJob = { classified: parseJsonField( message.message["classified"], "classified", ), createdAt: message.message["createdAt"] ?? new Date().toISOString(), derived: parseJsonField( message.message["derived"], "derived", ), latencyMs: parseOptionalNumberField(message.message["latencyMs"]), request: input.request, response: parseJsonField( message.message["output"], "output", ), }; if (process) { job.process = process; } return job; } export function calculatePercentile( values: number[], percentile: number, ): number | null { if (values.length === 0) { return null; } const sorted = [...values].sort((left, right) => left - right); const index = Math.min( sorted.length - 1, Math.max(0, Math.ceil(sorted.length * percentile) - 1), ); const selected = sorted[index]; return selected === undefined ? null : selected; } export function calculateEvalScoreDropPct( currentAverage: number | null, baselineAverage: number | null, ): number | null { if ( currentAverage === null || baselineAverage === null || baselineAverage <= 0 ) { return null; } return ((baselineAverage - currentAverage) / baselineAverage) * 100; } export function calculateLlmErrorRate( observations: EvalObservation[], ): number | null { if (observations.length === 0) { return null; } const errorCount = observations.filter( (observation) => observation.llmJudgeErrored, ).length; return errorCount / observations.length; } export class ShadowWorker { private archiveFailureCount = 0; private lastAlertCheckAt = 0; private lastArchiveAt = 0; private lastAttributionAt = 0; private lastDriftAt = 0; private lastEvalAggregateAt = 0; private lastPromoteAt = 0; private lastReviewAt = 0; private loopPromise: Promise | null = null; private readonly logger = getLogger(); private readonly recentEvalObservations: EvalObservation[] = []; private stopRequested = false; private warnedMissingMongo = false; constructor( private readonly garnetClient: GarnetClient, private readonly llmClient: LlmClient, private readonly conversationStore: ConversationStore, private readonly alertSender?: AlertSender, private readonly consumerName = `bg-worker-${process.pid}`, private readonly mongoConnection?: MongoConnectionLike | undefined, ) {} async start(): Promise { await this.ensureConsumerGroup(); if (this.loopPromise) { return; } this.stopRequested = false; this.loopPromise = this.runLoop(); } async stop(): Promise { this.stopRequested = true; if (!this.loopPromise) { return; } try { await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS); } catch (error) { this.logger.warn( { err: error }, "[ShadowWorker] Stop timed out while waiting for the loop to finish", ); } finally { this.loopPromise = null; } } private async ensureConsumerGroup(): Promise { try { await this.garnetClient.createConsumerGroup( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, ); } catch (error) { if (!String(error).includes("BUSYGROUP")) { throw error; } } } private async runLoop(): Promise { while (!this.stopRequested) { try { await this.runPeriodicChecksIfNeeded(); const recoveredPending = await this.recoverPendingOnce(); if (recoveredPending) { continue; } const streams = await this.garnetClient.readGroup( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, this.consumerName, ">", { blockMs: READ_BLOCK_MS, count: READ_COUNT, }, ); if (!streams) { continue; } for (const stream of streams) { for (const message of stream.messages) { if (this.stopRequested) { return; } await this.processMessage( message, message.deliveriesCounter ?? 1, ); } } } catch (error) { this.logger.error( { err: error }, "[ShadowWorker] Loop iteration failed", ); await delay(1_000); } } } private async recoverPendingOnce(): Promise { const pending = await this.garnetClient.getPendingStreamRange( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, PENDING_BATCH_SIZE, ); const recoverable = pending.filter( (entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS, ); if (recoverable.length === 0) { return false; } for (const entry of recoverable) { if (this.stopRequested) { break; } await this.recoverPendingEntry(entry); } return true; } private async recoverPendingEntry( entry: StreamPendingEntry, ): Promise { const nextDeliveryCount = entry.deliveriesCounter + 1; const claimed = await this.garnetClient.claimPendingStream( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, this.consumerName, PENDING_IDLE_MS, entry.id, { retryCount: nextDeliveryCount, }, ); const message = claimed[0]; if (!message) { return; } if (nextDeliveryCount > MAX_DELIVERIES) { await this.handleMaxDeliveries( message, nextDeliveryCount, "recovered entry exceeded max deliveries", ); return; } await this.processMessage(message, nextDeliveryCount); } private async processMessage( message: StreamEntry, deliveryCount: number, ): Promise { try { switch (getJobType(message) as BackgroundJobType) { case "shadow-eval": await this.processShadowEval(message); await this.acknowledge(message.id); return; case "summarize": await this.processSummarize(message); await this.acknowledge(message.id); return; default: this.logger.warn( { messageId: message.id, type: getJobType(message), }, "[ShadowWorker] Unknown background job type; acknowledging", ); await this.acknowledge(message.id); } } catch (error) { this.logger.warn( { err: error, deliveryCount, messageId: message.id, type: getJobType(message), }, "[ShadowWorker] Job processing failed", ); if (deliveryCount >= MAX_DELIVERIES) { await this.handleMaxDeliveries( message, deliveryCount, error instanceof Error ? error.message : String(error), ); } } } private async processShadowEval(message: StreamEntry): Promise { const job = parseShadowEvalJob(message); const abstention = getTraceAbstention(job.process); const responseTextForValidation = job.response.text.trim() ? job.response.text : abstention?.action.kind === "no-response" ? "[NO_RESPONSE]" : job.response.text; const validation = validateOnboardingResponse( responseTextForValidation, job.derived, { allowShortResponse: job.derived.allowShortResponse, }, ); const llmJudge = await judgeOnboardingWithLlm( this.llmClient, toEvalCase(job, message.id), job.response.text, toDerivedSnapshot(job.derived), ); const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall; const llmJudgeErrored = llmJudge.skipped && llmJudge.reason !== "empty-response" && llmJudge.reason !== "disabled"; const processJudge = job.process ? await this.judgeProcess(job, message.id) : undefined; this.recordEvalObservation({ llmJudgeErrored, llmOverall, recordedAt: Date.now(), responseLatencyMs: job.latencyMs ?? null, }); const resultFields: Record = { type: "shadow-eval-result", requestId: job.request.requestId, chatroomId: job.request.chatroomId, userId: job.request.userId, tier: String(job.derived.tier), intent: job.derived.intent.kind, phase: job.derived.phase.kind, responseMode: job.derived.responseMode, validatorStatus: validation.status, latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "", llmOverall: llmOverall === null ? "" : String(llmOverall), input: JSON.stringify({ request: job.request, }), output: JSON.stringify(job.response), derived: JSON.stringify(job.derived), classified: JSON.stringify(job.classified), validation: JSON.stringify(validation), llmJudge: JSON.stringify(llmJudge), evaluatedAt: new Date().toISOString(), }; if (abstention) { resultFields.abstentionAction = abstention.action.kind; resultFields.abstentionReason = abstention.reason ?? ""; resultFields.abstentionScore = String(abstention.score); } if (job.process) { resultFields.process = JSON.stringify(job.process); } if (processJudge) { resultFields.processJudge = JSON.stringify(processJudge); } await this.garnetClient.appendStream( EVAL_RESULTS_STREAM_KEY, resultFields, { maxLen: getEvalStreamMaxLen(), }, ); } private async judgeProcess( job: ShadowEvalJob, messageId: string, ): Promise { if (!job.process) { throw new Error("Missing process trace"); } const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process); if ( !getEvalProcessLlmJudgeEnabled() || Math.random() >= getEvalProcessLlmJudgeSamplingRate() ) { return ruleResult; } const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, { request: job.request, finalText: job.response.text, }); const merged: ProcessJudgeResult = { checks: [...ruleResult.checks, ...llmResult.checks], passedCount: ruleResult.passedCount + llmResult.passedCount, totalCount: ruleResult.totalCount + llmResult.totalCount, score: Number( ( (ruleResult.score * ruleResult.totalCount + llmResult.score * llmResult.totalCount) / Math.max(1, ruleResult.totalCount + llmResult.totalCount) ).toFixed(3), ), rationale: llmResult.rationale, }; const prmSource = llmResult.prmFeatures ? llmResult : ruleResult; if (prmSource.prmFeatures) { merged.prmFeatures = prmSource.prmFeatures; } if (prmSource.prmScore !== undefined) { merged.prmScore = prmSource.prmScore; } if (prmSource.rootCauseStep) { merged.rootCauseStep = prmSource.rootCauseStep; } if (prmSource.rootCauseScoreLoss !== undefined) { merged.rootCauseScoreLoss = prmSource.rootCauseScoreLoss; } return merged; } private async processSummarize(message: StreamEntry): Promise { const chatroomId = message.message["chatroomId"]; const userId = message.message["userId"]; if (!chatroomId?.trim() || !userId?.trim()) { throw new Error("Missing summarize job identifiers"); } const [history, existingSummary] = await Promise.all([ this.conversationStore.getHistory(chatroomId, userId), this.conversationStore.getSummary(chatroomId, userId), ]); if (history.length === 0) { return; } const summary = await summarizeConversation( this.llmClient, history, existingSummary ?? undefined, ); await this.conversationStore.setSummary(chatroomId, userId, summary); } private async handleMaxDeliveries( message: StreamEntry, deliveryCount: number, lastError: string, ): Promise { const type = getJobType(message); if (type === "summarize") { this.logger.warn( { deliveryCount, lastError, messageId: message.id, }, "[ShadowWorker] Dropping summarize job after max retries", ); await this.acknowledge(message.id); return; } await this.garnetClient.appendStream( BACKGROUND_DLQ_STREAM_KEY, { type, payload: JSON.stringify(message.message), retryCount: String(deliveryCount), lastError: lastError.slice(0, 500), originalId: message.id, }, { maxLen: 1_000, }, ); await this.acknowledge(message.id); } private acknowledge(id: string): Promise { return this.garnetClient.acknowledgeStream( BACKGROUND_STREAM_KEY, BACKGROUND_GROUP, id, ); } private recordEvalObservation(observation: EvalObservation): void { this.recentEvalObservations.push(observation); this.pruneOldObservations(Date.now()); } private pruneOldObservations(now: number): void { const cutoff = now - ALERT_WINDOW_MS; let removeCount = 0; for (const observation of this.recentEvalObservations) { if (observation.recordedAt >= cutoff) { break; } removeCount += 1; } if (removeCount > 0) { this.recentEvalObservations.splice(0, removeCount); } } private async runPeriodicChecksIfNeeded(): Promise { const now = Date.now(); await this.runAggregationTickIfNeeded(now); await this.runArchiveTickIfNeeded(now); await this.runAttributionTickIfNeeded(now); await this.runDriftTickIfNeeded(now); await this.runPromoteTickIfNeeded(now); await this.runReviewTickIfNeeded(now); if (!this.alertSender) { return; } if (now - this.lastAlertCheckAt < ALERT_TICK_MS) { return; } this.lastAlertCheckAt = now; this.pruneOldObservations(now); await this.checkLatencyAlert(now); await this.checkEvalScoreDropAlert(now); await this.checkLlmErrorRateAlert(now); await this.checkOpenCircuitAlerts(now); } private async runAggregationTickIfNeeded(now: number): Promise { if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) { return; } this.lastEvalAggregateAt = now; const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY); const result = await aggregateRecentEvalResults(this.garnetClient, { sinceId: sinceId ?? "0-0", nowMs: now, }); if (result.updatedSlices > 0) { this.logger.debug( { lastSeenId: result.lastSeenId, updatedSlices: result.updatedSlices, }, "[ShadowWorker] Aggregated shadow eval results", ); } } private warnMissingMongoOnce(): void { if (this.warnedMissingMongo) { return; } this.warnedMissingMongo = true; this.logger.warn( "[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled", ); } private async runArchiveTickIfNeeded(now: number): Promise { if (now - this.lastArchiveAt < getEvalArchiveTickMs()) { return; } this.lastArchiveAt = now; if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY); try { const result = await archiveEvalResults( this.garnetClient, this.mongoConnection, { sinceId: sinceId ?? "0-0", nowMs: now, }, ); const labelSinceId = await this.garnetClient.getString( EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY, ); const labelResult = await archiveReviewLabels( this.garnetClient, this.mongoConnection, { sinceId: labelSinceId ?? "0-0", nowMs: now, }, ); this.archiveFailureCount = 0; if (result.archivedCount > 0) { this.logger.info( { archivedCount: result.archivedCount, lastArchivedId: result.lastArchivedId, }, "[ShadowWorker] Archived shadow eval results", ); } if (labelResult.archivedCount > 0) { this.logger.info( { archivedCount: labelResult.archivedCount, lastArchivedId: labelResult.lastArchivedId, }, "[ShadowWorker] Archived review labels", ); } } catch (error) { this.archiveFailureCount += 1; this.logger.warn( { err: error, failureCount: this.archiveFailureCount, }, "[ShadowWorker] Failed to archive shadow eval results", ); if (this.archiveFailureCount >= 3 && this.alertSender) { await this.alertSender.sendAlert( { kind: "archive-stalled", metric: "shadow_eval.archive_failures", severity: "warn", suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.", threshold: 3, timestamp: new Date(now).toISOString(), value: this.archiveFailureCount, window: "consecutive", }, { dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled", }, ); } } } private async runDriftTickIfNeeded(now: number): Promise { if (now - this.lastDriftAt < getEvalDriftTickMs()) { return; } this.lastDriftAt = now; if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } if (!this.alertSender) { return; } const threshold = getAlertThresholdDriftPct(); const reports = await computeDriftReports(this.mongoConnection, { nowMs: now }); const significantReports = reports.filter((report) => report.significant); for (const report of significantReports) { const slice = sliceKeyToString(report.slice); await this.alertSender.sendAlert( { kind: "eval-drift", metric: "shadow_eval.drift.llm_overall_drop_pct", severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn", suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`, threshold, timestamp: new Date(now).toISOString(), value: report.dropPct, window: "24h_vs_7d", }, { dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`, }, ); } } private async runAttributionTickIfNeeded(now: number): Promise { if (now - this.lastAttributionAt < getEvalAttributionTickMs()) { return; } this.lastAttributionAt = now; if (!getEvalPrmEnabled()) { return; } if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } const model = loadPrmModel(getEvalPrmWeightsPath()); if (!model) { return; } const entries = await getEvalArchiveCollection(this.mongoConnection) .find({ process: { $exists: true }, }) .sort({ evaluatedAt: -1 }) .limit(getEvalArchiveBatchSize()) .toArray(); const result = await writeProcessAttributions( this.mongoConnection, entries, model, { nowMs: now }, ); if (result.attributedCount > 0) { this.logger.info( { attributedCount: result.attributedCount, modifiedCount: result.modifiedCount, upsertedCount: result.upsertedCount, }, "[ShadowWorker] Wrote process attribution entries", ); } } private async runPromoteTickIfNeeded(now: number): Promise { if (now - this.lastPromoteAt < getEvalPromoteTickMs()) { return; } this.lastPromoteAt = now; if (!this.mongoConnection) { this.warnMissingMongoOnce(); return; } const candidates = await selectPromotionCandidates(this.mongoConnection, { nowMs: now, }); if (candidates.length === 0) { return; } const result = await appendCandidatesToJsonl(candidates); this.logger.info( { appendedCount: result.appendedCount, outputPath: result.outputPath, }, "[ShadowWorker] Promoted shadow eval candidates", ); } private async runReviewTickIfNeeded(now: number): Promise { if (now - this.lastReviewAt < getEvalReviewTickMs()) { return; } this.lastReviewAt = now; const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY); const result = await routeReviewCandidates(this.garnetClient, { sinceId: sinceId ?? "0-0", nowMs: now, }); if (result.enqueuedCount > 0) { this.logger.info( { enqueuedCount: result.enqueuedCount, lastSeenId: result.lastSeenId, }, "[ShadowWorker] Routed eval results to review queue", ); } } private async checkLatencyAlert(now: number): Promise { const latencySamples = this.recentEvalObservations.flatMap((observation) => observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs], ); if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) { return; } const p95LatencyMs = calculatePercentile(latencySamples, 0.95); const threshold = getAlertThresholdP95LatencyMs(); if (p95LatencyMs === null || p95LatencyMs <= threshold) { return; } await this.alertSender.sendAlert({ kind: "latency", metric: "shadow_eval.p95_latency_ms", severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn", suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.", threshold, timestamp: new Date(now).toISOString(), value: p95LatencyMs, window: ALERT_WINDOW_LABEL, }); } private async checkEvalScoreDropAlert(now: number): Promise { if (!this.alertSender) { return; } const scoreSamples = this.recentEvalObservations.flatMap((observation) => observation.llmOverall === null ? [] : [observation.llmOverall], ); if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) { return; } const currentAverage = scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length; const baselineAverage = await this.loadBaselineOverallScore(); const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage); const threshold = getAlertThresholdEvalScoreDropPct(); if (dropPct === null || dropPct <= threshold) { return; } await this.alertSender.sendAlert({ kind: "eval-score-drop", metric: "shadow_eval.llm_overall_drop_pct", severity: dropPct >= threshold * 1.5 ? "critical" : "warn", suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.", threshold, timestamp: new Date(now).toISOString(), value: Number(dropPct.toFixed(3)), window: ALERT_WINDOW_LABEL, }); } private async checkLlmErrorRateAlert(now: number): Promise { if ( !this.alertSender || this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS ) { return; } const errorRate = calculateLlmErrorRate(this.recentEvalObservations); const threshold = getAlertThresholdLlmErrorRate(); if (errorRate === null || errorRate <= threshold) { return; } await this.alertSender.sendAlert({ kind: "llm-error-rate", metric: "shadow_eval.llm_error_rate", severity: errorRate >= threshold * 1.5 ? "critical" : "warn", suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.", threshold, timestamp: new Date(now).toISOString(), value: Number(errorRate.toFixed(4)), window: ALERT_WINDOW_LABEL, }); } private async checkOpenCircuitAlerts(now: number): Promise { if (!this.alertSender) { return; } const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*"); for (const key of circuitKeys) { const record = await this.garnetClient.getHash(key); if (record["state"] !== "open") { continue; } const endpointHash = key.split(":").at(-1) ?? "unknown"; const failureCount = Number(record["failureCount"]); await this.alertSender.sendAlert( { kind: "endpoint-down", metric: "llm.circuit.open", severity: "critical", suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.", threshold: 0, timestamp: new Date(now).toISOString(), value: Number.isFinite(failureCount) ? failureCount : 1, window: ALERT_WINDOW_LABEL, }, { dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`, }, ); } } private async loadBaselineOverallScore(): Promise { const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY); const redisParsed = parseBaselineOverallFromRaw(redisBaseline); if (redisParsed !== null) { return redisParsed; } const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE); if (!existsSync(filePath)) { return null; } try { const fileRaw = await readFile(filePath, "utf8"); return parseBaselineOverallFromRaw(fileRaw); } catch (error) { this.logger.warn( { err: error, filePath }, "[ShadowWorker] Failed to read baseline file", ); return null; } } } function parseBaselineOverallFromRaw(raw: string | null): number | null { if (!raw?.trim()) { return null; } try { const parsed = JSON.parse(raw) as Partial; const value = parsed.summary?.llmAverageOverall; return typeof value === "number" ? value : null; } catch { return null; } }