Files
lolympus-agent/src/eval/review/calibration.ts
T

206 lines
5.8 KiB
TypeScript

import {
type EvalArchiveEntry,
getEvalArchiveCollection,
type MongoConnectionLike,
} from "../archive.js";
import { queryArchivedReviewLabels } from "./label-archive.js";
import type { ReviewLabel } from "./queue.js";
import { readReviewLabels } from "./queue.js";
import type { GarnetClient } from "@common/garnet";
export interface CalibrationReport {
generatedAt: string;
meanGapOverall: number;
pairs: number;
pearsonByDim: {
accuracy: number;
concision: number;
friendliness: number;
scope: number;
};
pearsonOverall: number;
worstDisagreements: Array<{
evalStreamId: string;
humanScore: number;
llmOverall: number;
}>;
}
interface CalibrationPair {
evalStreamId: string;
humanScore: number;
llmAccuracy: number;
llmConcision: number;
llmFriendliness: number;
llmOverall: number;
llmScope: number;
}
function round(value: number): number {
return Number(value.toFixed(3));
}
function pearson(left: number[], right: number[]): number {
if (left.length < 2 || right.length < 2 || left.length !== right.length) {
return 0;
}
const leftMean = left.reduce((sum, value) => sum + value, 0) / left.length;
const rightMean = right.reduce((sum, value) => sum + value, 0) / right.length;
let numerator = 0;
let leftVariance = 0;
let rightVariance = 0;
for (let index = 0; index < left.length; index++) {
const leftDelta = (left[index] ?? 0) - leftMean;
const rightDelta = (right[index] ?? 0) - rightMean;
numerator += leftDelta * rightDelta;
leftVariance += leftDelta * leftDelta;
rightVariance += rightDelta * rightDelta;
}
const denominator = Math.sqrt(leftVariance * rightVariance);
return denominator === 0 ? 0 : round(numerator / denominator);
}
function toCalibrationPairs(
labels: ReviewLabel[],
archives: EvalArchiveEntry[],
): CalibrationPair[] {
const archiveByStreamId = new Map(
archives.map((entry) => [entry.streamId, entry]),
);
const pairs: CalibrationPair[] = [];
for (const label of labels) {
const archive = archiveByStreamId.get(label.evalStreamId);
if (!archive || archive.llmJudge.skipped) {
continue;
}
pairs.push({
evalStreamId: label.evalStreamId,
humanScore: label.humanScore,
llmAccuracy: archive.llmJudge.scores.accuracy ?? archive.llmJudge.scores.overall,
llmConcision: archive.llmJudge.scores.concision ?? archive.llmJudge.scores.overall,
llmFriendliness:
archive.llmJudge.scores.friendliness ?? archive.llmJudge.scores.overall,
llmOverall: archive.llmJudge.scores.overall,
llmScope: archive.llmJudge.scores.scope ?? archive.llmJudge.scores.overall,
});
}
return pairs;
}
function mergeLabels(
streamLabels: ReviewLabel[],
archivedLabels: ReviewLabel[],
): ReviewLabel[] {
const merged: ReviewLabel[] = [];
const seen = new Set<string>();
for (const label of [...streamLabels, ...archivedLabels]) {
const key = [
label.evalStreamId,
label.reviewedAt,
label.reviewedBy,
label.humanScore,
].join(":");
if (seen.has(key)) {
continue;
}
seen.add(key);
merged.push(label);
}
return merged;
}
export function computeCalibrationReportFromData(
labels: ReviewLabel[],
archives: EvalArchiveEntry[],
nowMs = Date.now(),
): CalibrationReport {
const pairs = toCalibrationPairs(labels, archives);
const humanScores = pairs.map((pair) => pair.humanScore);
const overallScores = pairs.map((pair) => pair.llmOverall);
const meanGapOverall =
pairs.length === 0
? 0
: round(
pairs.reduce(
(sum, pair) => sum + (pair.humanScore - pair.llmOverall),
0,
) / pairs.length,
);
return {
generatedAt: new Date(nowMs).toISOString(),
meanGapOverall,
pairs: pairs.length,
pearsonByDim: {
accuracy: pearson(humanScores, pairs.map((pair) => pair.llmAccuracy)),
concision: pearson(humanScores, pairs.map((pair) => pair.llmConcision)),
friendliness: pearson(
humanScores,
pairs.map((pair) => pair.llmFriendliness),
),
scope: pearson(humanScores, pairs.map((pair) => pair.llmScope)),
},
pearsonOverall: pearson(humanScores, overallScores),
worstDisagreements: pairs
.map((pair) => ({
evalStreamId: pair.evalStreamId,
humanScore: pair.humanScore,
llmOverall: pair.llmOverall,
}))
.sort(
(left, right) =>
Math.abs(right.humanScore - right.llmOverall) -
Math.abs(left.humanScore - left.llmOverall),
)
.slice(0, 10),
};
}
export async function computeCalibrationReport(
mongoConnection: MongoConnectionLike,
garnetClient: Pick<GarnetClient, "readStreamRange">,
options?: { sinceMs?: number | undefined },
): Promise<CalibrationReport> {
const nowMs = Date.now();
const [streamLabels, archivedLabels] = await Promise.all([
readReviewLabels(garnetClient),
queryArchivedReviewLabels(mongoConnection, {
sinceMs: options?.sinceMs,
nowMs,
}),
]);
const labels = mergeLabels(streamLabels, archivedLabels);
const cutoffMs =
options?.sinceMs === undefined ? null : nowMs - options.sinceMs;
const filteredLabels =
cutoffMs === null
? labels
: labels.filter((label) => {
const reviewedAt = Date.parse(label.reviewedAt);
return Number.isFinite(reviewedAt) && reviewedAt >= cutoffMs;
});
const ids = [...new Set(filteredLabels.map((label) => label.evalStreamId))];
const archives =
ids.length === 0
? []
: await getEvalArchiveCollection(mongoConnection)
.find({
streamId: {
$in: ids,
},
})
.sort({ evaluatedAt: -1 })
.limit(ids.length)
.toArray();
return computeCalibrationReportFromData(filteredLabels, archives);
}