import { type EvalArchiveEntry, getEvalArchiveCollection, type MongoConnectionLike, } from "../archive.js"; import { queryArchivedReviewLabels } from "./label-archive.js"; import type { ReviewLabel } from "./queue.js"; import { readReviewLabels } from "./queue.js"; import type { GarnetClient } from "@common/garnet"; export interface CalibrationReport { generatedAt: string; meanGapOverall: number; pairs: number; pearsonByDim: { accuracy: number; concision: number; friendliness: number; scope: number; }; pearsonOverall: number; worstDisagreements: Array<{ evalStreamId: string; humanScore: number; llmOverall: number; }>; } interface CalibrationPair { evalStreamId: string; humanScore: number; llmAccuracy: number; llmConcision: number; llmFriendliness: number; llmOverall: number; llmScope: number; } function round(value: number): number { return Number(value.toFixed(3)); } function pearson(left: number[], right: number[]): number { if (left.length < 2 || right.length < 2 || left.length !== right.length) { return 0; } const leftMean = left.reduce((sum, value) => sum + value, 0) / left.length; const rightMean = right.reduce((sum, value) => sum + value, 0) / right.length; let numerator = 0; let leftVariance = 0; let rightVariance = 0; for (let index = 0; index < left.length; index++) { const leftDelta = (left[index] ?? 0) - leftMean; const rightDelta = (right[index] ?? 0) - rightMean; numerator += leftDelta * rightDelta; leftVariance += leftDelta * leftDelta; rightVariance += rightDelta * rightDelta; } const denominator = Math.sqrt(leftVariance * rightVariance); return denominator === 0 ? 0 : round(numerator / denominator); } function toCalibrationPairs( labels: ReviewLabel[], archives: EvalArchiveEntry[], ): CalibrationPair[] { const archiveByStreamId = new Map( archives.map((entry) => [entry.streamId, entry]), ); const pairs: CalibrationPair[] = []; for (const label of labels) { const archive = archiveByStreamId.get(label.evalStreamId); if (!archive || archive.llmJudge.skipped) { continue; } pairs.push({ evalStreamId: label.evalStreamId, humanScore: label.humanScore, llmAccuracy: archive.llmJudge.scores.accuracy, llmConcision: archive.llmJudge.scores.concision, llmFriendliness: archive.llmJudge.scores.friendliness, llmOverall: archive.llmJudge.scores.overall, llmScope: archive.llmJudge.scores.scope, }); } return pairs; } function mergeLabels( streamLabels: ReviewLabel[], archivedLabels: ReviewLabel[], ): ReviewLabel[] { const merged: ReviewLabel[] = []; const seen = new Set(); for (const label of [...streamLabels, ...archivedLabels]) { const key = [ label.evalStreamId, label.reviewedAt, label.reviewedBy, label.humanScore, ].join(":"); if (seen.has(key)) { continue; } seen.add(key); merged.push(label); } return merged; } export function computeCalibrationReportFromData( labels: ReviewLabel[], archives: EvalArchiveEntry[], nowMs = Date.now(), ): CalibrationReport { const pairs = toCalibrationPairs(labels, archives); const humanScores = pairs.map((pair) => pair.humanScore); const overallScores = pairs.map((pair) => pair.llmOverall); const meanGapOverall = pairs.length === 0 ? 0 : round( pairs.reduce( (sum, pair) => sum + (pair.humanScore - pair.llmOverall), 0, ) / pairs.length, ); return { generatedAt: new Date(nowMs).toISOString(), meanGapOverall, pairs: pairs.length, pearsonByDim: { accuracy: pearson(humanScores, pairs.map((pair) => pair.llmAccuracy)), concision: pearson(humanScores, pairs.map((pair) => pair.llmConcision)), friendliness: pearson( humanScores, pairs.map((pair) => pair.llmFriendliness), ), scope: pearson(humanScores, pairs.map((pair) => pair.llmScope)), }, pearsonOverall: pearson(humanScores, overallScores), worstDisagreements: pairs .map((pair) => ({ evalStreamId: pair.evalStreamId, humanScore: pair.humanScore, llmOverall: pair.llmOverall, })) .sort( (left, right) => Math.abs(right.humanScore - right.llmOverall) - Math.abs(left.humanScore - left.llmOverall), ) .slice(0, 10), }; } export async function computeCalibrationReport( mongoConnection: MongoConnectionLike, garnetClient: Pick, options?: { sinceMs?: number | undefined }, ): Promise { const nowMs = Date.now(); const [streamLabels, archivedLabels] = await Promise.all([ readReviewLabels(garnetClient), queryArchivedReviewLabels(mongoConnection, { sinceMs: options?.sinceMs, nowMs, }), ]); const labels = mergeLabels(streamLabels, archivedLabels); const cutoffMs = options?.sinceMs === undefined ? null : nowMs - options.sinceMs; const filteredLabels = cutoffMs === null ? labels : labels.filter((label) => { const reviewedAt = Date.parse(label.reviewedAt); return Number.isFinite(reviewedAt) && reviewedAt >= cutoffMs; }); const ids = [...new Set(filteredLabels.map((label) => label.evalStreamId))]; const archives = ids.length === 0 ? [] : await getEvalArchiveCollection(mongoConnection) .find({ streamId: { $in: ids, }, }) .sort({ evaluatedAt: -1 }) .limit(ids.length) .toArray(); return computeCalibrationReportFromData(filteredLabels, archives); }