Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,151 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
archiveReviewLabels,
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION,
|
||||
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
||||
queryArchivedReviewLabels,
|
||||
type ReviewLabelArchiveEntry,
|
||||
} from "./label-archive.js";
|
||||
import { EVAL_REVIEW_LABELED_STREAM_KEY, type ReviewLabel } from "./queue.js";
|
||||
|
||||
class MemoryGarnet {
|
||||
readonly strings = new Map<string, string>();
|
||||
|
||||
constructor(private readonly entries: StreamEntry[]) {}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
_end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(EVAL_REVIEW_LABELED_STREAM_KEY);
|
||||
const filtered = start.startsWith("(")
|
||||
? this.entries.filter((entry) => entry.id > start.slice(1))
|
||||
: this.entries;
|
||||
return Promise.resolve(filtered.slice(0, options?.count));
|
||||
}
|
||||
|
||||
setString(key: string, value: string): Promise<void> {
|
||||
this.strings.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryCursor {
|
||||
constructor(private readonly entries: ReviewLabelArchiveEntry[]) {}
|
||||
|
||||
sort(): MemoryCursor {
|
||||
this.entries.sort(
|
||||
(left, right) => right.reviewedAt.getTime() - left.reviewedAt.getTime(),
|
||||
);
|
||||
return this;
|
||||
}
|
||||
|
||||
limit(value: number): MemoryCursor {
|
||||
this.entries.splice(value);
|
||||
return this;
|
||||
}
|
||||
|
||||
toArray(): Promise<ReviewLabelArchiveEntry[]> {
|
||||
return Promise.resolve(this.entries);
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryCollection {
|
||||
readonly docs = new Map<string, ReviewLabelArchiveEntry>();
|
||||
readonly indexes: Array<Record<string, unknown>> = [];
|
||||
|
||||
bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> {
|
||||
let upsertedCount = 0;
|
||||
for (const operation of operations) {
|
||||
const updateOne = (operation as {
|
||||
updateOne: {
|
||||
filter: { labelStreamId: string };
|
||||
update: { $setOnInsert: ReviewLabelArchiveEntry };
|
||||
};
|
||||
}).updateOne;
|
||||
const streamId = updateOne.filter.labelStreamId;
|
||||
if (!this.docs.has(streamId)) {
|
||||
this.docs.set(streamId, updateOne.update.$setOnInsert);
|
||||
upsertedCount += 1;
|
||||
}
|
||||
}
|
||||
return Promise.resolve({ upsertedCount });
|
||||
}
|
||||
|
||||
createIndex(keys: Record<string, unknown>): Promise<string> {
|
||||
this.indexes.push(keys);
|
||||
return Promise.resolve("idx");
|
||||
}
|
||||
|
||||
find(filter: Record<string, unknown>): MemoryCursor {
|
||||
const reviewedAt = filter["reviewedAt"] as
|
||||
| { $gte?: Date | undefined }
|
||||
| undefined;
|
||||
const entries = [...this.docs.values()].filter((entry) => {
|
||||
if (!reviewedAt?.$gte) {
|
||||
return true;
|
||||
}
|
||||
return entry.reviewedAt >= reviewedAt.$gte;
|
||||
});
|
||||
return new MemoryCursor(entries);
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryConnection {
|
||||
readonly collectionInstance = new MemoryCollection();
|
||||
|
||||
collection(name: string): MemoryCollection {
|
||||
expect(name).toBe(EVAL_REVIEW_LABEL_ARCHIVE_COLLECTION);
|
||||
return this.collectionInstance;
|
||||
}
|
||||
}
|
||||
|
||||
function streamEntry(id: string, label: ReviewLabel): StreamEntry {
|
||||
return {
|
||||
id,
|
||||
message: {
|
||||
type: "eval-review-label",
|
||||
evalStreamId: label.evalStreamId,
|
||||
reviewedAt: label.reviewedAt,
|
||||
reviewedBy: label.reviewedBy,
|
||||
payload: JSON.stringify(label),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("archiveReviewLabels", () => {
|
||||
it("archives label stream entries idempotently and queries them back", async () => {
|
||||
const label: ReviewLabel = {
|
||||
evalStreamId: "1-0",
|
||||
verdict: "fail",
|
||||
humanScore: 2,
|
||||
rationale: "wrong channel",
|
||||
reviewedBy: "tester",
|
||||
reviewedAt: "2026-05-01T00:01:00.000Z",
|
||||
};
|
||||
const garnet = new MemoryGarnet([streamEntry("1-0", label)]);
|
||||
const connection = new MemoryConnection();
|
||||
|
||||
const first = await archiveReviewLabels(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:02:00.000Z"),
|
||||
});
|
||||
const second = await archiveReviewLabels(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:02:00.000Z"),
|
||||
});
|
||||
const labels = await queryArchivedReviewLabels(connection);
|
||||
|
||||
expect(first).toEqual({ archivedCount: 1, lastArchivedId: "1-0" });
|
||||
expect(second).toEqual({ archivedCount: 0, lastArchivedId: "1-0" });
|
||||
expect(garnet.strings.get(EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY)).toBe("1-0");
|
||||
expect(connection.collectionInstance.indexes).toContainEqual({
|
||||
labelStreamId: 1,
|
||||
});
|
||||
expect(labels).toEqual([label]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user