Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,204 @@
|
||||
import {
|
||||
type EvalArchiveEntry,
|
||||
getEvalArchiveCollection,
|
||||
type MongoConnectionLike,
|
||||
} from "../archive.js";
|
||||
import { queryArchivedReviewLabels } from "./label-archive.js";
|
||||
import type { ReviewLabel } from "./queue.js";
|
||||
import { readReviewLabels } from "./queue.js";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
|
||||
export interface CalibrationReport {
|
||||
generatedAt: string;
|
||||
meanGapOverall: number;
|
||||
pairs: number;
|
||||
pearsonByDim: {
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
friendliness: number;
|
||||
scope: number;
|
||||
};
|
||||
pearsonOverall: number;
|
||||
worstDisagreements: Array<{
|
||||
evalStreamId: string;
|
||||
humanScore: number;
|
||||
llmOverall: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface CalibrationPair {
|
||||
evalStreamId: string;
|
||||
humanScore: number;
|
||||
llmAccuracy: number;
|
||||
llmConcision: number;
|
||||
llmFriendliness: number;
|
||||
llmOverall: number;
|
||||
llmScope: number;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function pearson(left: number[], right: number[]): number {
|
||||
if (left.length < 2 || right.length < 2 || left.length !== right.length) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const leftMean = left.reduce((sum, value) => sum + value, 0) / left.length;
|
||||
const rightMean = right.reduce((sum, value) => sum + value, 0) / right.length;
|
||||
let numerator = 0;
|
||||
let leftVariance = 0;
|
||||
let rightVariance = 0;
|
||||
|
||||
for (let index = 0; index < left.length; index++) {
|
||||
const leftDelta = (left[index] ?? 0) - leftMean;
|
||||
const rightDelta = (right[index] ?? 0) - rightMean;
|
||||
numerator += leftDelta * rightDelta;
|
||||
leftVariance += leftDelta * leftDelta;
|
||||
rightVariance += rightDelta * rightDelta;
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(leftVariance * rightVariance);
|
||||
return denominator === 0 ? 0 : round(numerator / denominator);
|
||||
}
|
||||
|
||||
function toCalibrationPairs(
|
||||
labels: ReviewLabel[],
|
||||
archives: EvalArchiveEntry[],
|
||||
): CalibrationPair[] {
|
||||
const archiveByStreamId = new Map(
|
||||
archives.map((entry) => [entry.streamId, entry]),
|
||||
);
|
||||
const pairs: CalibrationPair[] = [];
|
||||
|
||||
for (const label of labels) {
|
||||
const archive = archiveByStreamId.get(label.evalStreamId);
|
||||
if (!archive || archive.llmJudge.skipped) {
|
||||
continue;
|
||||
}
|
||||
|
||||
pairs.push({
|
||||
evalStreamId: label.evalStreamId,
|
||||
humanScore: label.humanScore,
|
||||
llmAccuracy: archive.llmJudge.scores.accuracy,
|
||||
llmConcision: archive.llmJudge.scores.concision,
|
||||
llmFriendliness: archive.llmJudge.scores.friendliness,
|
||||
llmOverall: archive.llmJudge.scores.overall,
|
||||
llmScope: archive.llmJudge.scores.scope,
|
||||
});
|
||||
}
|
||||
|
||||
return pairs;
|
||||
}
|
||||
|
||||
function mergeLabels(
|
||||
streamLabels: ReviewLabel[],
|
||||
archivedLabels: ReviewLabel[],
|
||||
): ReviewLabel[] {
|
||||
const merged: ReviewLabel[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const label of [...streamLabels, ...archivedLabels]) {
|
||||
const key = [
|
||||
label.evalStreamId,
|
||||
label.reviewedAt,
|
||||
label.reviewedBy,
|
||||
label.humanScore,
|
||||
].join(":");
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
merged.push(label);
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
|
||||
export function computeCalibrationReportFromData(
|
||||
labels: ReviewLabel[],
|
||||
archives: EvalArchiveEntry[],
|
||||
nowMs = Date.now(),
|
||||
): CalibrationReport {
|
||||
const pairs = toCalibrationPairs(labels, archives);
|
||||
const humanScores = pairs.map((pair) => pair.humanScore);
|
||||
const overallScores = pairs.map((pair) => pair.llmOverall);
|
||||
const meanGapOverall =
|
||||
pairs.length === 0
|
||||
? 0
|
||||
: round(
|
||||
pairs.reduce(
|
||||
(sum, pair) => sum + (pair.humanScore - pair.llmOverall),
|
||||
0,
|
||||
) / pairs.length,
|
||||
);
|
||||
|
||||
return {
|
||||
generatedAt: new Date(nowMs).toISOString(),
|
||||
meanGapOverall,
|
||||
pairs: pairs.length,
|
||||
pearsonByDim: {
|
||||
accuracy: pearson(humanScores, pairs.map((pair) => pair.llmAccuracy)),
|
||||
concision: pearson(humanScores, pairs.map((pair) => pair.llmConcision)),
|
||||
friendliness: pearson(
|
||||
humanScores,
|
||||
pairs.map((pair) => pair.llmFriendliness),
|
||||
),
|
||||
scope: pearson(humanScores, pairs.map((pair) => pair.llmScope)),
|
||||
},
|
||||
pearsonOverall: pearson(humanScores, overallScores),
|
||||
worstDisagreements: pairs
|
||||
.map((pair) => ({
|
||||
evalStreamId: pair.evalStreamId,
|
||||
humanScore: pair.humanScore,
|
||||
llmOverall: pair.llmOverall,
|
||||
}))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
Math.abs(right.humanScore - right.llmOverall) -
|
||||
Math.abs(left.humanScore - left.llmOverall),
|
||||
)
|
||||
.slice(0, 10),
|
||||
};
|
||||
}
|
||||
|
||||
export async function computeCalibrationReport(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
garnetClient: Pick<GarnetClient, "readStreamRange">,
|
||||
options?: { sinceMs?: number | undefined },
|
||||
): Promise<CalibrationReport> {
|
||||
const nowMs = Date.now();
|
||||
const [streamLabels, archivedLabels] = await Promise.all([
|
||||
readReviewLabels(garnetClient),
|
||||
queryArchivedReviewLabels(mongoConnection, {
|
||||
sinceMs: options?.sinceMs,
|
||||
nowMs,
|
||||
}),
|
||||
]);
|
||||
const labels = mergeLabels(streamLabels, archivedLabels);
|
||||
const cutoffMs =
|
||||
options?.sinceMs === undefined ? null : nowMs - options.sinceMs;
|
||||
const filteredLabels =
|
||||
cutoffMs === null
|
||||
? labels
|
||||
: labels.filter((label) => {
|
||||
const reviewedAt = Date.parse(label.reviewedAt);
|
||||
return Number.isFinite(reviewedAt) && reviewedAt >= cutoffMs;
|
||||
});
|
||||
const ids = [...new Set(filteredLabels.map((label) => label.evalStreamId))];
|
||||
const archives =
|
||||
ids.length === 0
|
||||
? []
|
||||
: await getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
streamId: {
|
||||
$in: ids,
|
||||
},
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(ids.length)
|
||||
.toArray();
|
||||
|
||||
return computeCalibrationReportFromData(filteredLabels, archives);
|
||||
}
|
||||
Reference in new issue
Block a user