191 lines
5.0 KiB
TypeScript
191 lines
5.0 KiB
TypeScript
import { getAlertThresholdDriftPct } from "../config.js";
|
|
import type {
|
|
EvalArchiveEntry,
|
|
MongoConnectionLike,
|
|
} from "./archive.js";
|
|
import { getEvalArchiveCollection } from "./archive.js";
|
|
import type { SliceKey } from "./aggregator.js";
|
|
|
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
const RECENT_WINDOW_MS = DAY_MS;
|
|
const BASELINE_WINDOW_MS = 7 * DAY_MS;
|
|
const MIN_SIGNIFICANT_RECENT_COUNT = 30;
|
|
|
|
export interface DriftReport {
|
|
slice: SliceKey;
|
|
recent24hAverage: number;
|
|
recent7dAverage: number;
|
|
dropPct: number;
|
|
significant: boolean;
|
|
recent24hCount: number;
|
|
recent7dCount: number;
|
|
}
|
|
|
|
interface DriftGroup {
|
|
baselineValues: number[];
|
|
recentValues: number[];
|
|
slice: SliceKey;
|
|
}
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(3));
|
|
}
|
|
|
|
function average(values: number[]): number | null {
|
|
if (values.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
}
|
|
|
|
function sliceKeyToString(slice: SliceKey): string {
|
|
return [
|
|
slice.tier,
|
|
slice.intent,
|
|
slice.phase,
|
|
slice.routingDecision ?? "-",
|
|
slice.abstentionAction ?? "-",
|
|
slice.abstentionReason ?? "-",
|
|
].join(":");
|
|
}
|
|
|
|
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
|
return {
|
|
tier: entry.tier,
|
|
intent: entry.derived.intent.kind,
|
|
phase: entry.derived.phase.kind,
|
|
...(entry.derived.routingDecision
|
|
? { routingDecision: entry.derived.routingDecision.kind }
|
|
: {}),
|
|
};
|
|
}
|
|
|
|
function asRecord(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function hasAbstained(entry: EvalArchiveEntry): boolean {
|
|
const process = asRecord(entry.process);
|
|
const steps = asRecord(process["steps"]);
|
|
const classify = asRecord(steps["classify"]);
|
|
const output = asRecord(classify["output"]);
|
|
const abstention = asRecord(output["abstention"]);
|
|
const action = asRecord(abstention["action"]);
|
|
const kind = action["kind"];
|
|
|
|
return typeof kind === "string" && kind !== "proceed";
|
|
}
|
|
|
|
function getMetricValue(
|
|
entry: EvalArchiveEntry,
|
|
metricKey: "abstentionRate" | "llmOverall" | "validatorPassRate",
|
|
): number | null {
|
|
if (metricKey === "abstentionRate") {
|
|
return hasAbstained(entry) ? 1 : 0;
|
|
}
|
|
|
|
if (metricKey === "validatorPassRate") {
|
|
return entry.validation.status === "pass" ? 1 : 0;
|
|
}
|
|
|
|
if (entry.llmJudge.skipped) {
|
|
return null;
|
|
}
|
|
|
|
return entry.llmJudge.scores.overall;
|
|
}
|
|
|
|
export function computeDriftReportsFromEntries(
|
|
entries: EvalArchiveEntry[],
|
|
options?: {
|
|
metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined;
|
|
nowMs?: number | undefined;
|
|
thresholdPct?: number | undefined;
|
|
},
|
|
): DriftReport[] {
|
|
const metricKey = options?.metricKey ?? "llmOverall";
|
|
const nowMs = options?.nowMs ?? Date.now();
|
|
const recentCutoff = nowMs - RECENT_WINDOW_MS;
|
|
const thresholdPct = options?.thresholdPct ?? getAlertThresholdDriftPct();
|
|
const groups = new Map<string, DriftGroup>();
|
|
|
|
for (const entry of entries) {
|
|
const value = getMetricValue(entry, metricKey);
|
|
if (value === null) {
|
|
continue;
|
|
}
|
|
|
|
const evaluatedAtMs = entry.evaluatedAt.getTime();
|
|
if (!Number.isFinite(evaluatedAtMs)) {
|
|
continue;
|
|
}
|
|
|
|
const slice = getSlice(entry);
|
|
const key = sliceKeyToString(slice);
|
|
const group =
|
|
groups.get(key) ??
|
|
{
|
|
baselineValues: [],
|
|
recentValues: [],
|
|
slice,
|
|
};
|
|
|
|
group.baselineValues.push(value);
|
|
if (evaluatedAtMs >= recentCutoff) {
|
|
group.recentValues.push(value);
|
|
}
|
|
groups.set(key, group);
|
|
}
|
|
|
|
return [...groups.values()].flatMap((group) => {
|
|
const recentAverage = average(group.recentValues);
|
|
const baselineAverage = average(group.baselineValues);
|
|
if (recentAverage === null || baselineAverage === null || baselineAverage <= 0) {
|
|
return [];
|
|
}
|
|
|
|
const dropPct = ((baselineAverage - recentAverage) / baselineAverage) * 100;
|
|
|
|
return [
|
|
{
|
|
slice: group.slice,
|
|
recent24hAverage: round(recentAverage),
|
|
recent7dAverage: round(baselineAverage),
|
|
dropPct: round(dropPct),
|
|
significant:
|
|
group.recentValues.length >= MIN_SIGNIFICANT_RECENT_COUNT &&
|
|
dropPct >= thresholdPct,
|
|
recent24hCount: group.recentValues.length,
|
|
recent7dCount: group.baselineValues.length,
|
|
},
|
|
];
|
|
});
|
|
}
|
|
|
|
export async function computeDriftReports(
|
|
mongoConnection: MongoConnectionLike,
|
|
options?: {
|
|
metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined;
|
|
nowMs?: number | undefined;
|
|
},
|
|
): Promise<DriftReport[]> {
|
|
const nowMs = options?.nowMs ?? Date.now();
|
|
const entries = await getEvalArchiveCollection(mongoConnection)
|
|
.find({
|
|
evaluatedAt: {
|
|
$gte: new Date(nowMs - BASELINE_WINDOW_MS),
|
|
},
|
|
})
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(50_000)
|
|
.toArray();
|
|
|
|
return computeDriftReportsFromEntries(entries, {
|
|
metricKey: options?.metricKey,
|
|
nowMs,
|
|
});
|
|
}
|