import { getAlertThresholdDriftPct } from "../config.js"; import type { EvalArchiveEntry, MongoConnectionLike, } from "./archive.js"; import { getEvalArchiveCollection } from "./archive.js"; import type { SliceKey } from "./aggregator.js"; const DAY_MS = 24 * 60 * 60 * 1000; const RECENT_WINDOW_MS = DAY_MS; const BASELINE_WINDOW_MS = 7 * DAY_MS; const MIN_SIGNIFICANT_RECENT_COUNT = 30; export interface DriftReport { slice: SliceKey; recent24hAverage: number; recent7dAverage: number; dropPct: number; significant: boolean; recent24hCount: number; recent7dCount: number; } interface DriftGroup { baselineValues: number[]; recentValues: number[]; slice: SliceKey; } function round(value: number): number { return Number(value.toFixed(3)); } function average(values: number[]): number | null { if (values.length === 0) { return null; } return values.reduce((sum, value) => sum + value, 0) / values.length; } function sliceKeyToString(slice: SliceKey): string { return [ slice.tier, slice.intent, slice.phase, slice.routingDecision ?? "-", slice.abstentionAction ?? "-", slice.abstentionReason ?? "-", ].join(":"); } function getSlice(entry: EvalArchiveEntry): SliceKey { return { tier: entry.tier, intent: entry.derived.intent.kind, phase: entry.derived.phase.kind, ...(entry.derived.routingDecision ? { routingDecision: entry.derived.routingDecision.kind } : {}), }; } function asRecord(value: unknown): Record { return typeof value === "object" && value !== null ? (value as Record) : {}; } function hasAbstained(entry: EvalArchiveEntry): boolean { const process = asRecord(entry.process); const steps = asRecord(process["steps"]); const classify = asRecord(steps["classify"]); const output = asRecord(classify["output"]); const abstention = asRecord(output["abstention"]); const action = asRecord(abstention["action"]); const kind = action["kind"]; return typeof kind === "string" && kind !== "proceed"; } function getMetricValue( entry: EvalArchiveEntry, metricKey: "abstentionRate" | "llmOverall" | "validatorPassRate", ): number | null { if (metricKey === "abstentionRate") { return hasAbstained(entry) ? 1 : 0; } if (metricKey === "validatorPassRate") { return entry.validation.status === "pass" ? 1 : 0; } if (entry.llmJudge.skipped) { return null; } return entry.llmJudge.scores.overall; } export function computeDriftReportsFromEntries( entries: EvalArchiveEntry[], options?: { metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined; nowMs?: number | undefined; thresholdPct?: number | undefined; }, ): DriftReport[] { const metricKey = options?.metricKey ?? "llmOverall"; const nowMs = options?.nowMs ?? Date.now(); const recentCutoff = nowMs - RECENT_WINDOW_MS; const thresholdPct = options?.thresholdPct ?? getAlertThresholdDriftPct(); const groups = new Map(); for (const entry of entries) { const value = getMetricValue(entry, metricKey); if (value === null) { continue; } const evaluatedAtMs = entry.evaluatedAt.getTime(); if (!Number.isFinite(evaluatedAtMs)) { continue; } const slice = getSlice(entry); const key = sliceKeyToString(slice); const group = groups.get(key) ?? { baselineValues: [], recentValues: [], slice, }; group.baselineValues.push(value); if (evaluatedAtMs >= recentCutoff) { group.recentValues.push(value); } groups.set(key, group); } return [...groups.values()].flatMap((group) => { const recentAverage = average(group.recentValues); const baselineAverage = average(group.baselineValues); if (recentAverage === null || baselineAverage === null || baselineAverage <= 0) { return []; } const dropPct = ((baselineAverage - recentAverage) / baselineAverage) * 100; return [ { slice: group.slice, recent24hAverage: round(recentAverage), recent7dAverage: round(baselineAverage), dropPct: round(dropPct), significant: group.recentValues.length >= MIN_SIGNIFICANT_RECENT_COUNT && dropPct >= thresholdPct, recent24hCount: group.recentValues.length, recent7dCount: group.baselineValues.length, }, ]; }); } export async function computeDriftReports( mongoConnection: MongoConnectionLike, options?: { metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined; nowMs?: number | undefined; }, ): Promise { const nowMs = options?.nowMs ?? Date.now(); const entries = await getEvalArchiveCollection(mongoConnection) .find({ evaluatedAt: { $gte: new Date(nowMs - BASELINE_WINDOW_MS), }, }) .sort({ evaluatedAt: -1 }) .limit(50_000) .toArray(); return computeDriftReportsFromEntries(entries, { metricKey: options?.metricKey, nowMs, }); }