189 lines
4.9 KiB
TypeScript
189 lines
4.9 KiB
TypeScript
import type { GarnetClient, StreamEntry } from "@common/garnet";
|
|
import { getEvalArchiveBatchSize } from "../../config.js";
|
|
import type { MongoConnectionLike } from "../archive.js";
|
|
import {
|
|
EVAL_REVIEW_LABELED_STREAM_KEY,
|
|
parseReviewLabelEntry,
|
|
type ReviewLabel,
|
|
} from "./queue.js";
|
|
|
|
export const EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY =
|
|
"lolympus-agent:eval:review:label-archive:cursor";
|
|
export const EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION = "eval_review_labels";
|
|
|
|
export interface ReviewLabelArchiveEntry
|
|
extends Omit<ReviewLabel, "reviewedAt"> {
|
|
labelStreamId: string;
|
|
reviewedAt: Date;
|
|
}
|
|
|
|
interface BulkWriteResultLike {
|
|
upsertedCount?: number | undefined;
|
|
}
|
|
|
|
interface FindCursorLike<T> {
|
|
limit(value: number): FindCursorLike<T>;
|
|
sort(value: Record<string, 1 | -1>): FindCursorLike<T>;
|
|
toArray(): Promise<T[]>;
|
|
}
|
|
|
|
interface ReviewLabelArchiveCollectionLike {
|
|
bulkWrite(
|
|
operations: unknown[],
|
|
options?: { ordered?: boolean | undefined },
|
|
): Promise<BulkWriteResultLike>;
|
|
createIndex(
|
|
keys: Record<string, unknown>,
|
|
options?: Record<string, unknown>,
|
|
): Promise<unknown>;
|
|
find(filter: Record<string, unknown>): FindCursorLike<ReviewLabelArchiveEntry>;
|
|
}
|
|
|
|
type LabelArchiveGarnetClient = Pick<
|
|
GarnetClient,
|
|
"readStreamRange" | "setString"
|
|
>;
|
|
|
|
function streamRangeStart(sinceId: string): string {
|
|
const trimmed = sinceId.trim();
|
|
if (!trimmed || trimmed === "0-0" || trimmed === "-") {
|
|
return "-";
|
|
}
|
|
return `(${trimmed}`;
|
|
}
|
|
|
|
function parseReviewedAt(value: string, nowMs: number): Date {
|
|
const parsed = new Date(value);
|
|
return Number.isFinite(parsed.getTime()) ? parsed : new Date(nowMs);
|
|
}
|
|
|
|
function toArchiveEntry(
|
|
streamEntry: StreamEntry,
|
|
nowMs: number,
|
|
): ReviewLabelArchiveEntry {
|
|
const label = parseReviewLabelEntry(streamEntry);
|
|
return {
|
|
...label,
|
|
labelStreamId: streamEntry.id,
|
|
reviewedAt: parseReviewedAt(label.reviewedAt, nowMs),
|
|
};
|
|
}
|
|
|
|
async function ensureReviewLabelArchiveIndexes(
|
|
collection: ReviewLabelArchiveCollectionLike,
|
|
): Promise<void> {
|
|
await Promise.all([
|
|
collection.createIndex({ labelStreamId: 1 }, { unique: true }),
|
|
collection.createIndex({ evalStreamId: 1 }),
|
|
collection.createIndex({ reviewedAt: -1 }),
|
|
]);
|
|
}
|
|
|
|
export function getReviewLabelArchiveCollection(
|
|
mongoConnection: MongoConnectionLike,
|
|
): ReviewLabelArchiveCollectionLike {
|
|
return mongoConnection.collection(
|
|
EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION,
|
|
) as ReviewLabelArchiveCollectionLike;
|
|
}
|
|
|
|
export async function archiveReviewLabels(
|
|
garnetClient: LabelArchiveGarnetClient,
|
|
mongoConnection: MongoConnectionLike,
|
|
options: {
|
|
batchSize?: number | undefined;
|
|
nowMs: number;
|
|
sinceId: string;
|
|
},
|
|
): Promise<{ archivedCount: number; lastArchivedId: string }> {
|
|
const collection = getReviewLabelArchiveCollection(mongoConnection);
|
|
await ensureReviewLabelArchiveIndexes(collection);
|
|
|
|
const entries = await garnetClient.readStreamRange(
|
|
EVAL_REVIEW_LABELED_STREAM_KEY,
|
|
streamRangeStart(options.sinceId),
|
|
"+",
|
|
{
|
|
count: options.batchSize ?? getEvalArchiveBatchSize(),
|
|
},
|
|
);
|
|
|
|
if (entries.length === 0) {
|
|
return {
|
|
archivedCount: 0,
|
|
lastArchivedId: options.sinceId,
|
|
};
|
|
}
|
|
|
|
const lastArchivedId = entries.at(-1)?.id ?? options.sinceId;
|
|
const archiveEntries = entries.flatMap((entry) => {
|
|
try {
|
|
return [toArchiveEntry(entry, options.nowMs)];
|
|
} catch {
|
|
return [];
|
|
}
|
|
});
|
|
|
|
let archivedCount = 0;
|
|
if (archiveEntries.length > 0) {
|
|
const result = await collection.bulkWrite(
|
|
archiveEntries.map((entry) => ({
|
|
updateOne: {
|
|
filter: { labelStreamId: entry.labelStreamId },
|
|
update: { $setOnInsert: entry },
|
|
upsert: true,
|
|
},
|
|
})),
|
|
{ ordered: false },
|
|
);
|
|
archivedCount = result.upsertedCount ?? 0;
|
|
}
|
|
|
|
await garnetClient.setString(
|
|
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
|
lastArchivedId,
|
|
);
|
|
|
|
return {
|
|
archivedCount,
|
|
lastArchivedId,
|
|
};
|
|
}
|
|
|
|
export async function queryArchivedReviewLabels(
|
|
mongoConnection: MongoConnectionLike,
|
|
options?: {
|
|
limit?: number | undefined;
|
|
nowMs?: number | undefined;
|
|
sinceMs?: number | undefined;
|
|
},
|
|
): Promise<ReviewLabel[]> {
|
|
const filter: Record<string, unknown> = {};
|
|
if (options?.sinceMs !== undefined) {
|
|
filter["reviewedAt"] = {
|
|
$gte: new Date((options.nowMs ?? Date.now()) - options.sinceMs),
|
|
};
|
|
}
|
|
|
|
const entries = await getReviewLabelArchiveCollection(mongoConnection)
|
|
.find(filter)
|
|
.sort({ reviewedAt: -1 })
|
|
.limit(options?.limit ?? 10_000)
|
|
.toArray();
|
|
|
|
return entries.map((entry) => {
|
|
const label: ReviewLabel = {
|
|
evalStreamId: entry.evalStreamId,
|
|
verdict: entry.verdict,
|
|
humanScore: entry.humanScore,
|
|
rationale: entry.rationale,
|
|
reviewedBy: entry.reviewedBy,
|
|
reviewedAt: entry.reviewedAt.toISOString(),
|
|
};
|
|
if (entry.betterResponse !== undefined) {
|
|
label.betterResponse = entry.betterResponse;
|
|
}
|
|
return label;
|
|
});
|
|
}
|