Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,158 @@
|
||||
import { getAlertThresholdDriftPct } from "../config.js";
|
||||
import type {
|
||||
EvalArchiveEntry,
|
||||
MongoConnectionLike,
|
||||
} from "./archive.js";
|
||||
import { getEvalArchiveCollection } from "./archive.js";
|
||||
import type { SliceKey } from "./aggregator.js";
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const RECENT_WINDOW_MS = DAY_MS;
|
||||
const BASELINE_WINDOW_MS = 7 * DAY_MS;
|
||||
const MIN_SIGNIFICANT_RECENT_COUNT = 30;
|
||||
|
||||
export interface DriftReport {
|
||||
slice: SliceKey;
|
||||
recent24hAverage: number;
|
||||
recent7dAverage: number;
|
||||
dropPct: number;
|
||||
significant: boolean;
|
||||
recent24hCount: number;
|
||||
recent7dCount: number;
|
||||
}
|
||||
|
||||
interface DriftGroup {
|
||||
baselineValues: number[];
|
||||
recentValues: number[];
|
||||
slice: SliceKey;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function average(values: number[]): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function sliceKeyToString(slice: SliceKey): string {
|
||||
return `${slice.tier}:${slice.intent}:${slice.phase}`;
|
||||
}
|
||||
|
||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
return {
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
};
|
||||
}
|
||||
|
||||
function getMetricValue(
|
||||
entry: EvalArchiveEntry,
|
||||
metricKey: "llmOverall" | "validatorPassRate",
|
||||
): number | null {
|
||||
if (metricKey === "validatorPassRate") {
|
||||
return entry.validation.status === "pass" ? 1 : 0;
|
||||
}
|
||||
|
||||
if (entry.llmJudge.skipped) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
export function computeDriftReportsFromEntries(
|
||||
entries: EvalArchiveEntry[],
|
||||
options?: {
|
||||
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
|
||||
nowMs?: number | undefined;
|
||||
thresholdPct?: number | undefined;
|
||||
},
|
||||
): DriftReport[] {
|
||||
const metricKey = options?.metricKey ?? "llmOverall";
|
||||
const nowMs = options?.nowMs ?? Date.now();
|
||||
const recentCutoff = nowMs - RECENT_WINDOW_MS;
|
||||
const thresholdPct = options?.thresholdPct ?? getAlertThresholdDriftPct();
|
||||
const groups = new Map<string, DriftGroup>();
|
||||
|
||||
for (const entry of entries) {
|
||||
const value = getMetricValue(entry, metricKey);
|
||||
if (value === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const evaluatedAtMs = entry.evaluatedAt.getTime();
|
||||
if (!Number.isFinite(evaluatedAtMs)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const slice = getSlice(entry);
|
||||
const key = sliceKeyToString(slice);
|
||||
const group =
|
||||
groups.get(key) ??
|
||||
{
|
||||
baselineValues: [],
|
||||
recentValues: [],
|
||||
slice,
|
||||
};
|
||||
|
||||
group.baselineValues.push(value);
|
||||
if (evaluatedAtMs >= recentCutoff) {
|
||||
group.recentValues.push(value);
|
||||
}
|
||||
groups.set(key, group);
|
||||
}
|
||||
|
||||
return [...groups.values()].flatMap((group) => {
|
||||
const recentAverage = average(group.recentValues);
|
||||
const baselineAverage = average(group.baselineValues);
|
||||
if (recentAverage === null || baselineAverage === null || baselineAverage <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const dropPct = ((baselineAverage - recentAverage) / baselineAverage) * 100;
|
||||
|
||||
return [
|
||||
{
|
||||
slice: group.slice,
|
||||
recent24hAverage: round(recentAverage),
|
||||
recent7dAverage: round(baselineAverage),
|
||||
dropPct: round(dropPct),
|
||||
significant:
|
||||
group.recentValues.length >= MIN_SIGNIFICANT_RECENT_COUNT &&
|
||||
dropPct >= thresholdPct,
|
||||
recent24hCount: group.recentValues.length,
|
||||
recent7dCount: group.baselineValues.length,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export async function computeDriftReports(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
|
||||
nowMs?: number | undefined;
|
||||
},
|
||||
): Promise<DriftReport[]> {
|
||||
const nowMs = options?.nowMs ?? Date.now();
|
||||
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
evaluatedAt: {
|
||||
$gte: new Date(nowMs - BASELINE_WINDOW_MS),
|
||||
},
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(50_000)
|
||||
.toArray();
|
||||
|
||||
return computeDriftReportsFromEntries(entries, {
|
||||
metricKey: options?.metricKey,
|
||||
nowMs,
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user