206 lines
5.8 KiB
TypeScript
206 lines
5.8 KiB
TypeScript
import {
|
|
type EvalArchiveEntry,
|
|
getEvalArchiveCollection,
|
|
type MongoConnectionLike,
|
|
} from "../archive.js";
|
|
import { queryArchivedReviewLabels } from "./label-archive.js";
|
|
import type { ReviewLabel } from "./queue.js";
|
|
import { readReviewLabels } from "./queue.js";
|
|
import type { GarnetClient } from "@common/garnet";
|
|
|
|
export interface CalibrationReport {
|
|
generatedAt: string;
|
|
meanGapOverall: number;
|
|
pairs: number;
|
|
pearsonByDim: {
|
|
accuracy: number;
|
|
concision: number;
|
|
friendliness: number;
|
|
scope: number;
|
|
};
|
|
pearsonOverall: number;
|
|
worstDisagreements: Array<{
|
|
evalStreamId: string;
|
|
humanScore: number;
|
|
llmOverall: number;
|
|
}>;
|
|
}
|
|
|
|
interface CalibrationPair {
|
|
evalStreamId: string;
|
|
humanScore: number;
|
|
llmAccuracy: number;
|
|
llmConcision: number;
|
|
llmFriendliness: number;
|
|
llmOverall: number;
|
|
llmScope: number;
|
|
}
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(3));
|
|
}
|
|
|
|
function pearson(left: number[], right: number[]): number {
|
|
if (left.length < 2 || right.length < 2 || left.length !== right.length) {
|
|
return 0;
|
|
}
|
|
|
|
const leftMean = left.reduce((sum, value) => sum + value, 0) / left.length;
|
|
const rightMean = right.reduce((sum, value) => sum + value, 0) / right.length;
|
|
let numerator = 0;
|
|
let leftVariance = 0;
|
|
let rightVariance = 0;
|
|
|
|
for (let index = 0; index < left.length; index++) {
|
|
const leftDelta = (left[index] ?? 0) - leftMean;
|
|
const rightDelta = (right[index] ?? 0) - rightMean;
|
|
numerator += leftDelta * rightDelta;
|
|
leftVariance += leftDelta * leftDelta;
|
|
rightVariance += rightDelta * rightDelta;
|
|
}
|
|
|
|
const denominator = Math.sqrt(leftVariance * rightVariance);
|
|
return denominator === 0 ? 0 : round(numerator / denominator);
|
|
}
|
|
|
|
function toCalibrationPairs(
|
|
labels: ReviewLabel[],
|
|
archives: EvalArchiveEntry[],
|
|
): CalibrationPair[] {
|
|
const archiveByStreamId = new Map(
|
|
archives.map((entry) => [entry.streamId, entry]),
|
|
);
|
|
const pairs: CalibrationPair[] = [];
|
|
|
|
for (const label of labels) {
|
|
const archive = archiveByStreamId.get(label.evalStreamId);
|
|
if (!archive || archive.llmJudge.skipped) {
|
|
continue;
|
|
}
|
|
|
|
pairs.push({
|
|
evalStreamId: label.evalStreamId,
|
|
humanScore: label.humanScore,
|
|
llmAccuracy: archive.llmJudge.scores.accuracy ?? archive.llmJudge.scores.overall,
|
|
llmConcision: archive.llmJudge.scores.concision ?? archive.llmJudge.scores.overall,
|
|
llmFriendliness:
|
|
archive.llmJudge.scores.friendliness ?? archive.llmJudge.scores.overall,
|
|
llmOverall: archive.llmJudge.scores.overall,
|
|
llmScope: archive.llmJudge.scores.scope ?? archive.llmJudge.scores.overall,
|
|
});
|
|
}
|
|
|
|
return pairs;
|
|
}
|
|
|
|
function mergeLabels(
|
|
streamLabels: ReviewLabel[],
|
|
archivedLabels: ReviewLabel[],
|
|
): ReviewLabel[] {
|
|
const merged: ReviewLabel[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (const label of [...streamLabels, ...archivedLabels]) {
|
|
const key = [
|
|
label.evalStreamId,
|
|
label.reviewedAt,
|
|
label.reviewedBy,
|
|
label.humanScore,
|
|
].join(":");
|
|
if (seen.has(key)) {
|
|
continue;
|
|
}
|
|
seen.add(key);
|
|
merged.push(label);
|
|
}
|
|
|
|
return merged;
|
|
}
|
|
|
|
export function computeCalibrationReportFromData(
|
|
labels: ReviewLabel[],
|
|
archives: EvalArchiveEntry[],
|
|
nowMs = Date.now(),
|
|
): CalibrationReport {
|
|
const pairs = toCalibrationPairs(labels, archives);
|
|
const humanScores = pairs.map((pair) => pair.humanScore);
|
|
const overallScores = pairs.map((pair) => pair.llmOverall);
|
|
const meanGapOverall =
|
|
pairs.length === 0
|
|
? 0
|
|
: round(
|
|
pairs.reduce(
|
|
(sum, pair) => sum + (pair.humanScore - pair.llmOverall),
|
|
0,
|
|
) / pairs.length,
|
|
);
|
|
|
|
return {
|
|
generatedAt: new Date(nowMs).toISOString(),
|
|
meanGapOverall,
|
|
pairs: pairs.length,
|
|
pearsonByDim: {
|
|
accuracy: pearson(humanScores, pairs.map((pair) => pair.llmAccuracy)),
|
|
concision: pearson(humanScores, pairs.map((pair) => pair.llmConcision)),
|
|
friendliness: pearson(
|
|
humanScores,
|
|
pairs.map((pair) => pair.llmFriendliness),
|
|
),
|
|
scope: pearson(humanScores, pairs.map((pair) => pair.llmScope)),
|
|
},
|
|
pearsonOverall: pearson(humanScores, overallScores),
|
|
worstDisagreements: pairs
|
|
.map((pair) => ({
|
|
evalStreamId: pair.evalStreamId,
|
|
humanScore: pair.humanScore,
|
|
llmOverall: pair.llmOverall,
|
|
}))
|
|
.sort(
|
|
(left, right) =>
|
|
Math.abs(right.humanScore - right.llmOverall) -
|
|
Math.abs(left.humanScore - left.llmOverall),
|
|
)
|
|
.slice(0, 10),
|
|
};
|
|
}
|
|
|
|
export async function computeCalibrationReport(
|
|
mongoConnection: MongoConnectionLike,
|
|
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
|
options?: { sinceMs?: number | undefined },
|
|
): Promise<CalibrationReport> {
|
|
const nowMs = Date.now();
|
|
const [streamLabels, archivedLabels] = await Promise.all([
|
|
readReviewLabels(garnetClient),
|
|
queryArchivedReviewLabels(mongoConnection, {
|
|
sinceMs: options?.sinceMs,
|
|
nowMs,
|
|
}),
|
|
]);
|
|
const labels = mergeLabels(streamLabels, archivedLabels);
|
|
const cutoffMs =
|
|
options?.sinceMs === undefined ? null : nowMs - options.sinceMs;
|
|
const filteredLabels =
|
|
cutoffMs === null
|
|
? labels
|
|
: labels.filter((label) => {
|
|
const reviewedAt = Date.parse(label.reviewedAt);
|
|
return Number.isFinite(reviewedAt) && reviewedAt >= cutoffMs;
|
|
});
|
|
const ids = [...new Set(filteredLabels.map((label) => label.evalStreamId))];
|
|
const archives =
|
|
ids.length === 0
|
|
? []
|
|
: await getEvalArchiveCollection(mongoConnection)
|
|
.find({
|
|
streamId: {
|
|
$in: ids,
|
|
},
|
|
})
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(ids.length)
|
|
.toArray();
|
|
|
|
return computeCalibrationReportFromData(filteredLabels, archives);
|
|
}
|