Files
lolympus-agent/src/eval/shadow-worker.ts
T

1157 lines
31 KiB
TypeScript

import { existsSync } from "node:fs";
import { readFile } from "node:fs/promises";
import { resolve } from "node:path";
import type { AgentRequest, AgentResponse } from "@agent/types";
import type {
GarnetClient,
StreamEntry,
StreamPendingEntry,
} from "@common/garnet";
import { summarizeConversation } from "@conv/summarizer";
import type { LlmClient } from "@llm/llm-client";
import { getLogger } from "@obs/logger";
import type { AlertSender } from "@obs/alert";
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import type {
ClassifiedIntent,
DerivedState,
} from "@skills/onboarding/state/onboarding-state";
import type { ConversationStore } from "@conv/conversation-store";
import {
getAlertThresholdDriftPct,
getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs,
getEvalAggTickMs,
getEvalArchiveTickMs,
getEvalArchiveBatchSize,
getEvalAttributionTickMs,
getEvalDriftTickMs,
getEvalProcessLlmJudgeEnabled,
getEvalProcessLlmJudgeSamplingRate,
getEvalPrmEnabled,
getEvalPrmWeightsPath,
getEvalPromoteTickMs,
getEvalReviewTickMs,
getEvalStreamMaxLen,
} from "../config.js";
import {
aggregateRecentEvalResults,
EVAL_AGG_CURSOR_KEY,
sliceKeyToString,
} from "./aggregator.js";
import {
archiveEvalResults,
EVAL_ARCHIVE_CURSOR_KEY,
getEvalArchiveCollection,
type MongoConnectionLike,
} from "./archive.js";
import { computeDriftReports } from "./drift.js";
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
import { writeProcessAttributions } from "./judges/attribution.js";
import { loadPrmModel } from "./judges/prm.js";
import {
judgeProcessRule,
judgeProcessWithLlm,
} from "./judges/process-judge.js";
import {
appendCandidatesToJsonl,
selectPromotionCandidates,
} from "./promote.js";
import {
EVAL_REVIEW_CURSOR_KEY,
routeReviewCandidates,
} from "./review/queue.js";
import {
archiveReviewLabels,
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
} from "./review/label-archive.js";
import type {
EvalBaselineFile,
EvalCaseDefinition,
EvalDerivedSnapshot,
ProcessJudgeResult,
} from "./types.js";
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
const BACKGROUND_GROUP = "bg-worker";
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
const ALERT_WINDOW_MS = 5 * 60_000;
const ALERT_WINDOW_LABEL = "5m";
const ALERT_TICK_MS = 60_000;
const MAX_DELIVERIES = 3;
const MIN_ERROR_RATE_OBSERVATIONS = 5;
const MIN_LATENCY_OBSERVATIONS = 5;
const MIN_SCORE_OBSERVATIONS = 3;
const PENDING_IDLE_MS = 1_000;
const READ_BLOCK_MS = 5_000;
const READ_COUNT = 10;
const PENDING_BATCH_SIZE = 20;
const STOP_TIMEOUT_MS = 10_000;
type BackgroundJobType = "shadow-eval" | "summarize";
interface ShadowEvalJob {
classified: ClassifiedIntent;
createdAt: string;
derived: DerivedState;
latencyMs?: number | undefined;
process?: ProcessTraceSnapshot | undefined;
request: AgentRequest;
response: AgentResponse;
}
interface EvalObservation {
llmJudgeErrored: boolean;
llmOverall: number | null;
recordedAt: number;
responseLatencyMs: number | null;
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
function promiseWithTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
): Promise<T> {
return Promise.race([
promise,
new Promise<T>((_, reject) => {
setTimeout(() => {
reject(new Error(`Timed out after ${timeoutMs}ms`));
}, timeoutMs);
}),
]);
}
function getJobType(message: StreamEntry): string {
return message.message["type"] ?? "unknown";
}
function parseOptionalNumberField(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : undefined;
}
function parseJsonField<T>(
value: string | undefined,
fieldName: string,
): T {
if (!value?.trim()) {
throw new Error(`Missing ${fieldName} field`);
}
return JSON.parse(value) as T;
}
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
if (!value?.trim()) {
return undefined;
}
return JSON.parse(value) as T;
}
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
return {
...(derived.abstentionScore !== undefined
? { abstentionScore: derived.abstentionScore }
: {}),
allowShortResponse: derived.allowShortResponse,
intent: derived.intent.kind,
phase: derived.phase.kind,
responseMode: derived.responseMode,
...(derived.routingDecision
? { routingDecision: derived.routingDecision }
: {}),
shouldRespond: derived.shouldRespond,
suggestedMaxTokens: derived.suggestedMaxTokens,
tier: derived.tier,
};
}
function getTraceAbstention(process: ProcessTraceSnapshot | undefined):
| NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
| undefined {
return process?.steps.classify?.output.abstention;
}
function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition {
return {
id: `shadow:${messageId}`,
lineNumber: 0,
input: {
request: job.request,
},
expected: {},
};
}
function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
const input = parseJsonField<{ request: AgentRequest }>(
message.message["input"],
"input",
);
const process = parseOptionalJsonField<ProcessTraceSnapshot>(
message.message["process"],
);
const job: ShadowEvalJob = {
classified: parseJsonField<ClassifiedIntent>(
message.message["classified"],
"classified",
),
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
derived: parseJsonField<DerivedState>(
message.message["derived"],
"derived",
),
latencyMs: parseOptionalNumberField(message.message["latencyMs"]),
request: input.request,
response: parseJsonField<AgentResponse>(
message.message["output"],
"output",
),
};
if (process) {
job.process = process;
}
return job;
}
export function calculatePercentile(
values: number[],
percentile: number,
): number | null {
if (values.length === 0) {
return null;
}
const sorted = [...values].sort((left, right) => left - right);
const index = Math.min(
sorted.length - 1,
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
);
const selected = sorted[index];
return selected === undefined ? null : selected;
}
export function calculateEvalScoreDropPct(
currentAverage: number | null,
baselineAverage: number | null,
): number | null {
if (
currentAverage === null ||
baselineAverage === null ||
baselineAverage <= 0
) {
return null;
}
return ((baselineAverage - currentAverage) / baselineAverage) * 100;
}
export function calculateLlmErrorRate(
observations: EvalObservation[],
): number | null {
if (observations.length === 0) {
return null;
}
const errorCount = observations.filter(
(observation) => observation.llmJudgeErrored,
).length;
return errorCount / observations.length;
}
export class ShadowWorker {
private archiveFailureCount = 0;
private lastAlertCheckAt = 0;
private lastArchiveAt = 0;
private lastAttributionAt = 0;
private lastDriftAt = 0;
private lastEvalAggregateAt = 0;
private lastPromoteAt = 0;
private lastReviewAt = 0;
private loopPromise: Promise<void> | null = null;
private readonly logger = getLogger();
private readonly recentEvalObservations: EvalObservation[] = [];
private stopRequested = false;
private warnedMissingMongo = false;
constructor(
private readonly garnetClient: GarnetClient,
private readonly llmClient: LlmClient,
private readonly conversationStore: ConversationStore,
private readonly alertSender?: AlertSender,
private readonly consumerName = `bg-worker-${process.pid}`,
private readonly mongoConnection?: MongoConnectionLike | undefined,
) {}
async start(): Promise<void> {
await this.ensureConsumerGroup();
if (this.loopPromise) {
return;
}
this.stopRequested = false;
this.loopPromise = this.runLoop();
}
async stop(): Promise<void> {
this.stopRequested = true;
if (!this.loopPromise) {
return;
}
try {
await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS);
} catch (error) {
this.logger.warn(
{ err: error },
"[ShadowWorker] Stop timed out while waiting for the loop to finish",
);
} finally {
this.loopPromise = null;
}
}
private async ensureConsumerGroup(): Promise<void> {
try {
await this.garnetClient.createConsumerGroup(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
);
} catch (error) {
if (!String(error).includes("BUSYGROUP")) {
throw error;
}
}
}
private async runLoop(): Promise<void> {
while (!this.stopRequested) {
try {
await this.runPeriodicChecksIfNeeded();
const recoveredPending = await this.recoverPendingOnce();
if (recoveredPending) {
continue;
}
const streams = await this.garnetClient.readGroup(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
this.consumerName,
">",
{
blockMs: READ_BLOCK_MS,
count: READ_COUNT,
},
);
if (!streams) {
continue;
}
for (const stream of streams) {
for (const message of stream.messages) {
if (this.stopRequested) {
return;
}
await this.processMessage(
message,
message.deliveriesCounter ?? 1,
);
}
}
} catch (error) {
this.logger.error(
{ err: error },
"[ShadowWorker] Loop iteration failed",
);
await delay(1_000);
}
}
}
private async recoverPendingOnce(): Promise<boolean> {
const pending = await this.garnetClient.getPendingStreamRange(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
PENDING_BATCH_SIZE,
);
const recoverable = pending.filter(
(entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS,
);
if (recoverable.length === 0) {
return false;
}
for (const entry of recoverable) {
if (this.stopRequested) {
break;
}
await this.recoverPendingEntry(entry);
}
return true;
}
private async recoverPendingEntry(
entry: StreamPendingEntry,
): Promise<void> {
const nextDeliveryCount = entry.deliveriesCounter + 1;
const claimed = await this.garnetClient.claimPendingStream(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
this.consumerName,
PENDING_IDLE_MS,
entry.id,
{
retryCount: nextDeliveryCount,
},
);
const message = claimed[0];
if (!message) {
return;
}
if (nextDeliveryCount > MAX_DELIVERIES) {
await this.handleMaxDeliveries(
message,
nextDeliveryCount,
"recovered entry exceeded max deliveries",
);
return;
}
await this.processMessage(message, nextDeliveryCount);
}
private async processMessage(
message: StreamEntry,
deliveryCount: number,
): Promise<void> {
try {
switch (getJobType(message) as BackgroundJobType) {
case "shadow-eval":
await this.processShadowEval(message);
await this.acknowledge(message.id);
return;
case "summarize":
await this.processSummarize(message);
await this.acknowledge(message.id);
return;
default:
this.logger.warn(
{
messageId: message.id,
type: getJobType(message),
},
"[ShadowWorker] Unknown background job type; acknowledging",
);
await this.acknowledge(message.id);
}
} catch (error) {
this.logger.warn(
{
err: error,
deliveryCount,
messageId: message.id,
type: getJobType(message),
},
"[ShadowWorker] Job processing failed",
);
if (deliveryCount >= MAX_DELIVERIES) {
await this.handleMaxDeliveries(
message,
deliveryCount,
error instanceof Error ? error.message : String(error),
);
}
}
}
private async processShadowEval(message: StreamEntry): Promise<void> {
const job = parseShadowEvalJob(message);
const abstention = getTraceAbstention(job.process);
const responseTextForValidation =
job.response.text.trim()
? job.response.text
: abstention?.action.kind === "no-response"
? "[NO_RESPONSE]"
: job.response.text;
const validation = validateOnboardingResponse(
responseTextForValidation,
job.derived,
{
allowShortResponse: job.derived.allowShortResponse,
},
);
const llmJudge = await judgeOnboardingWithLlm(
this.llmClient,
toEvalCase(job, message.id),
job.response.text,
toDerivedSnapshot(job.derived),
);
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
const llmJudgeErrored =
llmJudge.skipped &&
llmJudge.reason !== "empty-response" &&
llmJudge.reason !== "disabled";
const processJudge = job.process
? await this.judgeProcess(job, message.id)
: undefined;
this.recordEvalObservation({
llmJudgeErrored,
llmOverall,
recordedAt: Date.now(),
responseLatencyMs: job.latencyMs ?? null,
});
const resultFields: Record<string, string> = {
type: "shadow-eval-result",
requestId: job.request.requestId,
chatroomId: job.request.chatroomId,
userId: job.request.userId,
tier: String(job.derived.tier),
intent: job.derived.intent.kind,
phase: job.derived.phase.kind,
responseMode: job.derived.responseMode,
validatorStatus: validation.status,
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
llmOverall: llmOverall === null ? "" : String(llmOverall),
input: JSON.stringify({
request: job.request,
}),
output: JSON.stringify(job.response),
derived: JSON.stringify(job.derived),
classified: JSON.stringify(job.classified),
validation: JSON.stringify(validation),
llmJudge: JSON.stringify(llmJudge),
evaluatedAt: new Date().toISOString(),
};
if (abstention) {
resultFields.abstentionAction = abstention.action.kind;
resultFields.abstentionReason = abstention.reason ?? "";
resultFields.abstentionScore = String(abstention.score);
}
if (job.process) {
resultFields.process = JSON.stringify(job.process);
}
if (processJudge) {
resultFields.processJudge = JSON.stringify(processJudge);
}
await this.garnetClient.appendStream(
EVAL_RESULTS_STREAM_KEY,
resultFields,
{
maxLen: getEvalStreamMaxLen(),
},
);
}
private async judgeProcess(
job: ShadowEvalJob,
messageId: string,
): Promise<ProcessJudgeResult> {
if (!job.process) {
throw new Error("Missing process trace");
}
const ruleResult = judgeProcessRule(toEvalCase(job, messageId).expected, job.process);
if (
!getEvalProcessLlmJudgeEnabled() ||
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
) {
return ruleResult;
}
const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, {
request: job.request,
finalText: job.response.text,
});
const merged: ProcessJudgeResult = {
checks: [...ruleResult.checks, ...llmResult.checks],
passedCount: ruleResult.passedCount + llmResult.passedCount,
totalCount: ruleResult.totalCount + llmResult.totalCount,
score: Number(
(
(ruleResult.score * ruleResult.totalCount +
llmResult.score * llmResult.totalCount) /
Math.max(1, ruleResult.totalCount + llmResult.totalCount)
).toFixed(3),
),
rationale: llmResult.rationale,
};
const prmSource = llmResult.prmFeatures ? llmResult : ruleResult;
if (prmSource.prmFeatures) {
merged.prmFeatures = prmSource.prmFeatures;
}
if (prmSource.prmScore !== undefined) {
merged.prmScore = prmSource.prmScore;
}
if (prmSource.rootCauseStep) {
merged.rootCauseStep = prmSource.rootCauseStep;
}
if (prmSource.rootCauseScoreLoss !== undefined) {
merged.rootCauseScoreLoss = prmSource.rootCauseScoreLoss;
}
return merged;
}
private async processSummarize(message: StreamEntry): Promise<void> {
const chatroomId = message.message["chatroomId"];
const userId = message.message["userId"];
if (!chatroomId?.trim() || !userId?.trim()) {
throw new Error("Missing summarize job identifiers");
}
const [history, existingSummary] = await Promise.all([
this.conversationStore.getHistory(chatroomId, userId),
this.conversationStore.getSummary(chatroomId, userId),
]);
if (history.length === 0) {
return;
}
const summary = await summarizeConversation(
this.llmClient,
history,
existingSummary ?? undefined,
);
await this.conversationStore.setSummary(chatroomId, userId, summary);
}
private async handleMaxDeliveries(
message: StreamEntry,
deliveryCount: number,
lastError: string,
): Promise<void> {
const type = getJobType(message);
if (type === "summarize") {
this.logger.warn(
{
deliveryCount,
lastError,
messageId: message.id,
},
"[ShadowWorker] Dropping summarize job after max retries",
);
await this.acknowledge(message.id);
return;
}
await this.garnetClient.appendStream(
BACKGROUND_DLQ_STREAM_KEY,
{
type,
payload: JSON.stringify(message.message),
retryCount: String(deliveryCount),
lastError: lastError.slice(0, 500),
originalId: message.id,
},
{
maxLen: 1_000,
},
);
await this.acknowledge(message.id);
}
private acknowledge(id: string): Promise<number> {
return this.garnetClient.acknowledgeStream(
BACKGROUND_STREAM_KEY,
BACKGROUND_GROUP,
id,
);
}
private recordEvalObservation(observation: EvalObservation): void {
this.recentEvalObservations.push(observation);
this.pruneOldObservations(Date.now());
}
private pruneOldObservations(now: number): void {
const cutoff = now - ALERT_WINDOW_MS;
let removeCount = 0;
for (const observation of this.recentEvalObservations) {
if (observation.recordedAt >= cutoff) {
break;
}
removeCount += 1;
}
if (removeCount > 0) {
this.recentEvalObservations.splice(0, removeCount);
}
}
private async runPeriodicChecksIfNeeded(): Promise<void> {
const now = Date.now();
await this.runAggregationTickIfNeeded(now);
await this.runArchiveTickIfNeeded(now);
await this.runAttributionTickIfNeeded(now);
await this.runDriftTickIfNeeded(now);
await this.runPromoteTickIfNeeded(now);
await this.runReviewTickIfNeeded(now);
if (!this.alertSender) {
return;
}
if (now - this.lastAlertCheckAt < ALERT_TICK_MS) {
return;
}
this.lastAlertCheckAt = now;
this.pruneOldObservations(now);
await this.checkLatencyAlert(now);
await this.checkEvalScoreDropAlert(now);
await this.checkLlmErrorRateAlert(now);
await this.checkOpenCircuitAlerts(now);
}
private async runAggregationTickIfNeeded(now: number): Promise<void> {
if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) {
return;
}
this.lastEvalAggregateAt = now;
const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY);
const result = await aggregateRecentEvalResults(this.garnetClient, {
sinceId: sinceId ?? "0-0",
nowMs: now,
});
if (result.updatedSlices > 0) {
this.logger.debug(
{
lastSeenId: result.lastSeenId,
updatedSlices: result.updatedSlices,
},
"[ShadowWorker] Aggregated shadow eval results",
);
}
}
private warnMissingMongoOnce(): void {
if (this.warnedMissingMongo) {
return;
}
this.warnedMissingMongo = true;
this.logger.warn(
"[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled",
);
}
private async runArchiveTickIfNeeded(now: number): Promise<void> {
if (now - this.lastArchiveAt < getEvalArchiveTickMs()) {
return;
}
this.lastArchiveAt = now;
if (!this.mongoConnection) {
this.warnMissingMongoOnce();
return;
}
const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY);
try {
const result = await archiveEvalResults(
this.garnetClient,
this.mongoConnection,
{
sinceId: sinceId ?? "0-0",
nowMs: now,
},
);
const labelSinceId = await this.garnetClient.getString(
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
);
const labelResult = await archiveReviewLabels(
this.garnetClient,
this.mongoConnection,
{
sinceId: labelSinceId ?? "0-0",
nowMs: now,
},
);
this.archiveFailureCount = 0;
if (result.archivedCount > 0) {
this.logger.info(
{
archivedCount: result.archivedCount,
lastArchivedId: result.lastArchivedId,
},
"[ShadowWorker] Archived shadow eval results",
);
}
if (labelResult.archivedCount > 0) {
this.logger.info(
{
archivedCount: labelResult.archivedCount,
lastArchivedId: labelResult.lastArchivedId,
},
"[ShadowWorker] Archived review labels",
);
}
} catch (error) {
this.archiveFailureCount += 1;
this.logger.warn(
{
err: error,
failureCount: this.archiveFailureCount,
},
"[ShadowWorker] Failed to archive shadow eval results",
);
if (this.archiveFailureCount >= 3 && this.alertSender) {
await this.alertSender.sendAlert(
{
kind: "archive-stalled",
metric: "shadow_eval.archive_failures",
severity: "warn",
suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.",
threshold: 3,
timestamp: new Date(now).toISOString(),
value: this.archiveFailureCount,
window: "consecutive",
},
{
dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled",
},
);
}
}
}
private async runDriftTickIfNeeded(now: number): Promise<void> {
if (now - this.lastDriftAt < getEvalDriftTickMs()) {
return;
}
this.lastDriftAt = now;
if (!this.mongoConnection) {
this.warnMissingMongoOnce();
return;
}
if (!this.alertSender) {
return;
}
const threshold = getAlertThresholdDriftPct();
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
const significantReports = reports.filter((report) => report.significant);
for (const report of significantReports) {
const slice = sliceKeyToString(report.slice);
await this.alertSender.sendAlert(
{
kind: "eval-drift",
metric: "shadow_eval.drift.llm_overall_drop_pct",
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
threshold,
timestamp: new Date(now).toISOString(),
value: report.dropPct,
window: "24h_vs_7d",
},
{
dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`,
},
);
}
}
private async runAttributionTickIfNeeded(now: number): Promise<void> {
if (now - this.lastAttributionAt < getEvalAttributionTickMs()) {
return;
}
this.lastAttributionAt = now;
if (!getEvalPrmEnabled()) {
return;
}
if (!this.mongoConnection) {
this.warnMissingMongoOnce();
return;
}
const model = loadPrmModel(getEvalPrmWeightsPath());
if (!model) {
return;
}
const entries = await getEvalArchiveCollection(this.mongoConnection)
.find({
process: { $exists: true },
})
.sort({ evaluatedAt: -1 })
.limit(getEvalArchiveBatchSize())
.toArray();
const result = await writeProcessAttributions(
this.mongoConnection,
entries,
model,
{ nowMs: now },
);
if (result.attributedCount > 0) {
this.logger.info(
{
attributedCount: result.attributedCount,
modifiedCount: result.modifiedCount,
upsertedCount: result.upsertedCount,
},
"[ShadowWorker] Wrote process attribution entries",
);
}
}
private async runPromoteTickIfNeeded(now: number): Promise<void> {
if (now - this.lastPromoteAt < getEvalPromoteTickMs()) {
return;
}
this.lastPromoteAt = now;
if (!this.mongoConnection) {
this.warnMissingMongoOnce();
return;
}
const candidates = await selectPromotionCandidates(this.mongoConnection, {
nowMs: now,
});
if (candidates.length === 0) {
return;
}
const result = await appendCandidatesToJsonl(candidates);
this.logger.info(
{
appendedCount: result.appendedCount,
outputPath: result.outputPath,
},
"[ShadowWorker] Promoted shadow eval candidates",
);
}
private async runReviewTickIfNeeded(now: number): Promise<void> {
if (now - this.lastReviewAt < getEvalReviewTickMs()) {
return;
}
this.lastReviewAt = now;
const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY);
const result = await routeReviewCandidates(this.garnetClient, {
sinceId: sinceId ?? "0-0",
nowMs: now,
});
if (result.enqueuedCount > 0) {
this.logger.info(
{
enqueuedCount: result.enqueuedCount,
lastSeenId: result.lastSeenId,
},
"[ShadowWorker] Routed eval results to review queue",
);
}
}
private async checkLatencyAlert(now: number): Promise<void> {
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
);
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
return;
}
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
const threshold = getAlertThresholdP95LatencyMs();
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: p95LatencyMs,
window: ALERT_WINDOW_LABEL,
});
}
private async checkEvalScoreDropAlert(now: number): Promise<void> {
if (!this.alertSender) {
return;
}
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
observation.llmOverall === null ? [] : [observation.llmOverall],
);
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
return;
}
const currentAverage =
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
const baselineAverage = await this.loadBaselineOverallScore();
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
const threshold = getAlertThresholdEvalScoreDropPct();
if (dropPct === null || dropPct <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "eval-score-drop",
metric: "shadow_eval.llm_overall_drop_pct",
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(dropPct.toFixed(3)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkLlmErrorRateAlert(now: number): Promise<void> {
if (
!this.alertSender ||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
) {
return;
}
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
const threshold = getAlertThresholdLlmErrorRate();
if (errorRate === null || errorRate <= threshold) {
return;
}
await this.alertSender.sendAlert({
kind: "llm-error-rate",
metric: "shadow_eval.llm_error_rate",
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(errorRate.toFixed(4)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkOpenCircuitAlerts(now: number): Promise<void> {
if (!this.alertSender) {
return;
}
const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*");
for (const key of circuitKeys) {
const record = await this.garnetClient.getHash(key);
if (record["state"] !== "open") {
continue;
}
const endpointHash = key.split(":").at(-1) ?? "unknown";
const failureCount = Number(record["failureCount"]);
await this.alertSender.sendAlert(
{
kind: "endpoint-down",
metric: "llm.circuit.open",
severity: "critical",
suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.",
threshold: 0,
timestamp: new Date(now).toISOString(),
value: Number.isFinite(failureCount) ? failureCount : 1,
window: ALERT_WINDOW_LABEL,
},
{
dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`,
},
);
}
}
private async loadBaselineOverallScore(): Promise<number | null> {
const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY);
const redisParsed = parseBaselineOverallFromRaw(redisBaseline);
if (redisParsed !== null) {
return redisParsed;
}
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
if (!existsSync(filePath)) {
return null;
}
try {
const fileRaw = await readFile(filePath, "utf8");
return parseBaselineOverallFromRaw(fileRaw);
} catch (error) {
this.logger.warn(
{ err: error, filePath },
"[ShadowWorker] Failed to read baseline file",
);
return null;
}
}
}
function parseBaselineOverallFromRaw(raw: string | null): number | null {
if (!raw?.trim()) {
return null;
}
try {
const parsed = JSON.parse(raw) as Partial<EvalBaselineFile>;
const value = parsed.summary?.llmAverageOverall;
return typeof value === "number" ? value : null;
} catch {
return null;
}
}