Files
lolympus-agent/src/eval/review/label-archive.ts
T

189 lines
4.9 KiB
TypeScript

import type { GarnetClient, StreamEntry } from "@common/garnet";
import { getEvalArchiveBatchSize } from "../../config.js";
import type { MongoConnectionLike } from "../archive.js";
import {
EVAL_REVIEW_LABELED_STREAM_KEY,
parseReviewLabelEntry,
type ReviewLabel,
} from "./queue.js";
export const EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY =
"lolympus-agent:eval:review:label-archive:cursor";
export const EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION = "eval_review_labels";
export interface ReviewLabelArchiveEntry
extends Omit<ReviewLabel, "reviewedAt"> {
labelStreamId: string;
reviewedAt: Date;
}
interface BulkWriteResultLike {
upsertedCount?: number | undefined;
}
interface FindCursorLike<T> {
limit(value: number): FindCursorLike<T>;
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
toArray(): Promise<T[]>;
}
interface ReviewLabelArchiveCollectionLike {
bulkWrite(
operations: unknown[],
options?: { ordered?: boolean | undefined },
): Promise<BulkWriteResultLike>;
createIndex(
keys: Record<string, unknown>,
options?: Record<string, unknown>,
): Promise<unknown>;
find(filter: Record<string, unknown>): FindCursorLike<ReviewLabelArchiveEntry>;
}
type LabelArchiveGarnetClient = Pick<
GarnetClient,
"readStreamRange" | "setString"
>;
function streamRangeStart(sinceId: string): string {
const trimmed = sinceId.trim();
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
return "-";
}
return `(${trimmed}`;
}
function parseReviewedAt(value: string, nowMs: number): Date {
const parsed = new Date(value);
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
}
function toArchiveEntry(
streamEntry: StreamEntry,
nowMs: number,
): ReviewLabelArchiveEntry {
const label = parseReviewLabelEntry(streamEntry);
return {
...label,
labelStreamId: streamEntry.id,
reviewedAt: parseReviewedAt(label.reviewedAt, nowMs),
};
}
async function ensureReviewLabelArchiveIndexes(
collection: ReviewLabelArchiveCollectionLike,
): Promise<void> {
await Promise.all([
collection.createIndex({ labelStreamId: 1 }, { unique: true }),
collection.createIndex({ evalStreamId: 1 }),
collection.createIndex({ reviewedAt: -1 }),
]);
}
export function getReviewLabelArchiveCollection(
mongoConnection: MongoConnectionLike,
): ReviewLabelArchiveCollectionLike {
return mongoConnection.collection(
EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION,
) as ReviewLabelArchiveCollectionLike;
}
export async function archiveReviewLabels(
garnetClient: LabelArchiveGarnetClient,
mongoConnection: MongoConnectionLike,
options: {
batchSize?: number | undefined;
nowMs: number;
sinceId: string;
},
): Promise<{ archivedCount: number; lastArchivedId: string }> {
const collection = getReviewLabelArchiveCollection(mongoConnection);
await ensureReviewLabelArchiveIndexes(collection);
const entries = await garnetClient.readStreamRange(
EVAL_REVIEW_LABELED_STREAM_KEY,
streamRangeStart(options.sinceId),
"+",
{
count: options.batchSize ?? getEvalArchiveBatchSize(),
},
);
if (entries.length === 0) {
return {
archivedCount: 0,
lastArchivedId: options.sinceId,
};
}
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
const archiveEntries = entries.flatMap((entry) => {
try {
return [toArchiveEntry(entry, options.nowMs)];
} catch {
return [];
}
});
let archivedCount = 0;
if (archiveEntries.length > 0) {
const result = await collection.bulkWrite(
archiveEntries.map((entry) => ({
updateOne: {
filter: { labelStreamId: entry.labelStreamId },
update: { $setOnInsert: entry },
upsert: true,
},
})),
{ ordered: false },
);
archivedCount = result.upsertedCount ?? 0;
}
await garnetClient.setString(
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
lastArchivedId,
);
return {
archivedCount,
lastArchivedId,
};
}
export async function queryArchivedReviewLabels(
mongoConnection: MongoConnectionLike,
options?: {
limit?: number | undefined;
nowMs?: number | undefined;
sinceMs?: number | undefined;
},
): Promise<ReviewLabel[]> {
const filter: Record<string, unknown> = {};
if (options?.sinceMs !== undefined) {
filter["reviewedAt"] = {
$gte: new Date((options.nowMs ?? Date.now()) - options.sinceMs),
};
}
const entries = await getReviewLabelArchiveCollection(mongoConnection)
.find(filter)
.sort({ reviewedAt: -1 })
.limit(options?.limit ?? 10_000)
.toArray();
return entries.map((entry) => {
const label: ReviewLabel = {
evalStreamId: entry.evalStreamId,
verdict: entry.verdict,
humanScore: entry.humanScore,
rationale: entry.rationale,
reviewedBy: entry.reviewedBy,
reviewedAt: entry.reviewedAt.toISOString(),
};
if (entry.betterResponse !== undefined) {
label.betterResponse = entry.betterResponse;
}
return label;
});
}