Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,188 @@
|
||||
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
||||
import { getEvalArchiveBatchSize } from "../../config.js";
|
||||
import type { MongoConnectionLike } from "../archive.js";
|
||||
import {
|
||||
EVAL_REVIEW_LABELED_STREAM_KEY,
|
||||
parseReviewLabelEntry,
|
||||
type ReviewLabel,
|
||||
} from "./queue.js";
|
||||
|
||||
export const EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY =
|
||||
"lolympus-agent:eval:review:label-archive:cursor";
|
||||
export const EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION = "eval_review_labels";
|
||||
|
||||
export interface ReviewLabelArchiveEntry
|
||||
extends Omit<ReviewLabel, "reviewedAt"> {
|
||||
labelStreamId: string;
|
||||
reviewedAt: Date;
|
||||
}
|
||||
|
||||
interface BulkWriteResultLike {
|
||||
upsertedCount?: number | undefined;
|
||||
}
|
||||
|
||||
interface FindCursorLike<T> {
|
||||
limit(value: number): FindCursorLike<T>;
|
||||
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
|
||||
toArray(): Promise<T[]>;
|
||||
}
|
||||
|
||||
interface ReviewLabelArchiveCollectionLike {
|
||||
bulkWrite(
|
||||
operations: unknown[],
|
||||
options?: { ordered?: boolean | undefined },
|
||||
): Promise<BulkWriteResultLike>;
|
||||
createIndex(
|
||||
keys: Record<string, unknown>,
|
||||
options?: Record<string, unknown>,
|
||||
): Promise<unknown>;
|
||||
find(filter: Record<string, unknown>): FindCursorLike<ReviewLabelArchiveEntry>;
|
||||
}
|
||||
|
||||
type LabelArchiveGarnetClient = Pick<
|
||||
GarnetClient,
|
||||
"readStreamRange" | "setString"
|
||||
>;
|
||||
|
||||
function streamRangeStart(sinceId: string): string {
|
||||
const trimmed = sinceId.trim();
|
||||
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
||||
return "-";
|
||||
}
|
||||
return `(${trimmed}`;
|
||||
}
|
||||
|
||||
function parseReviewedAt(value: string, nowMs: number): Date {
|
||||
const parsed = new Date(value);
|
||||
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
|
||||
}
|
||||
|
||||
function toArchiveEntry(
|
||||
streamEntry: StreamEntry,
|
||||
nowMs: number,
|
||||
): ReviewLabelArchiveEntry {
|
||||
const label = parseReviewLabelEntry(streamEntry);
|
||||
return {
|
||||
...label,
|
||||
labelStreamId: streamEntry.id,
|
||||
reviewedAt: parseReviewedAt(label.reviewedAt, nowMs),
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureReviewLabelArchiveIndexes(
|
||||
collection: ReviewLabelArchiveCollectionLike,
|
||||
): Promise<void> {
|
||||
await Promise.all([
|
||||
collection.createIndex({ labelStreamId: 1 }, { unique: true }),
|
||||
collection.createIndex({ evalStreamId: 1 }),
|
||||
collection.createIndex({ reviewedAt: -1 }),
|
||||
]);
|
||||
}
|
||||
|
||||
export function getReviewLabelArchiveCollection(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
): ReviewLabelArchiveCollectionLike {
|
||||
return mongoConnection.collection(
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION,
|
||||
) as ReviewLabelArchiveCollectionLike;
|
||||
}
|
||||
|
||||
export async function archiveReviewLabels(
|
||||
garnetClient: LabelArchiveGarnetClient,
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options: {
|
||||
batchSize?: number | undefined;
|
||||
nowMs: number;
|
||||
sinceId: string;
|
||||
},
|
||||
): Promise<{ archivedCount: number; lastArchivedId: string }> {
|
||||
const collection = getReviewLabelArchiveCollection(mongoConnection);
|
||||
await ensureReviewLabelArchiveIndexes(collection);
|
||||
|
||||
const entries = await garnetClient.readStreamRange(
|
||||
EVAL_REVIEW_LABELED_STREAM_KEY,
|
||||
streamRangeStart(options.sinceId),
|
||||
"+",
|
||||
{
|
||||
count: options.batchSize ?? getEvalArchiveBatchSize(),
|
||||
},
|
||||
);
|
||||
|
||||
if (entries.length === 0) {
|
||||
return {
|
||||
archivedCount: 0,
|
||||
lastArchivedId: options.sinceId,
|
||||
};
|
||||
}
|
||||
|
||||
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
|
||||
const archiveEntries = entries.flatMap((entry) => {
|
||||
try {
|
||||
return [toArchiveEntry(entry, options.nowMs)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
|
||||
let archivedCount = 0;
|
||||
if (archiveEntries.length > 0) {
|
||||
const result = await collection.bulkWrite(
|
||||
archiveEntries.map((entry) => ({
|
||||
updateOne: {
|
||||
filter: { labelStreamId: entry.labelStreamId },
|
||||
update: { $setOnInsert: entry },
|
||||
upsert: true,
|
||||
},
|
||||
})),
|
||||
{ ordered: false },
|
||||
);
|
||||
archivedCount = result.upsertedCount ?? 0;
|
||||
}
|
||||
|
||||
await garnetClient.setString(
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
lastArchivedId,
|
||||
);
|
||||
|
||||
return {
|
||||
archivedCount,
|
||||
lastArchivedId,
|
||||
};
|
||||
}
|
||||
|
||||
export async function queryArchivedReviewLabels(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
limit?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
sinceMs?: number | undefined;
|
||||
},
|
||||
): Promise<ReviewLabel[]> {
|
||||
const filter: Record<string, unknown> = {};
|
||||
if (options?.sinceMs !== undefined) {
|
||||
filter["reviewedAt"] = {
|
||||
$gte: new Date((options.nowMs ?? Date.now()) - options.sinceMs),
|
||||
};
|
||||
}
|
||||
|
||||
const entries = await getReviewLabelArchiveCollection(mongoConnection)
|
||||
.find(filter)
|
||||
.sort({ reviewedAt: -1 })
|
||||
.limit(options?.limit ?? 10_000)
|
||||
.toArray();
|
||||
|
||||
return entries.map((entry) => {
|
||||
const label: ReviewLabel = {
|
||||
evalStreamId: entry.evalStreamId,
|
||||
verdict: entry.verdict,
|
||||
humanScore: entry.humanScore,
|
||||
rationale: entry.rationale,
|
||||
reviewedBy: entry.reviewedBy,
|
||||
reviewedAt: entry.reviewedAt.toISOString(),
|
||||
};
|
||||
if (entry.betterResponse !== undefined) {
|
||||
label.betterResponse = entry.betterResponse;
|
||||
}
|
||||
return label;
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user