Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit a900016be4
60 files changed
+8689 -142

No files matched your search

+163
View File
@@ -0,0 +1,163 @@
import { describe, expect, it } from "vitest";
import type { StreamEntry } from "@common/garnet";
import {
archiveEvalResults,
EVAL_ARCHIVE_CURSOR_KEY,
EVAL_ARCHIVE_COLLECTION,
type EvalArchiveEntry,
} from "./archive.js";
import { EVAL_RESULTS_STREAM_KEY } from "./aggregator.js";
class MemoryGarnet {
readonly strings = new Map<string, string>();
constructor(private readonly entries: StreamEntry[]) {}
readStreamRange(
key: string,
start: string,
_end: string,
options?: { count?: number | undefined },
): Promise<StreamEntry[]> {
expect(key).toBe(EVAL_RESULTS_STREAM_KEY);
const filtered = start.startsWith("(")
? this.entries.filter((entry) => entry.id > start.slice(1))
: this.entries;
return Promise.resolve(filtered.slice(0, options?.count));
}
setString(key: string, value: string): Promise<void> {
this.strings.set(key, value);
return Promise.resolve();
}
}
class MemoryCollection {
readonly docs = new Map<string, EvalArchiveEntry>();
readonly indexes: Array<Record<string, unknown>> = [];
bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> {
let upsertedCount = 0;
for (const operation of operations) {
const updateOne = (operation as {
updateOne: {
filter: { streamId: string };
update: { $setOnInsert: EvalArchiveEntry };
};
}).updateOne;
const streamId = updateOne.filter.streamId;
if (!this.docs.has(streamId)) {
this.docs.set(streamId, updateOne.update.$setOnInsert);
upsertedCount += 1;
}
}
return Promise.resolve({ upsertedCount });
}
createIndex(keys: Record<string, unknown>): Promise<string> {
this.indexes.push(keys);
return Promise.resolve("idx");
}
find(): never {
throw new Error("not used");
}
}
class MemoryConnection {
readonly collectionInstance = new MemoryCollection();
collection(name: string): MemoryCollection {
expect(name).toBe(EVAL_ARCHIVE_COLLECTION);
return this.collectionInstance;
}
}
function streamEntry(id: string): StreamEntry {
return {
id,
message: {
input: JSON.stringify({
request: {
requestId: "req-1",
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "닉",
trigger: "user-message",
userMessage: "상태 알려줘",
},
}),
output: JSON.stringify({
requestId: "req-1",
text: "사진 인증만 남았어요.",
confidence: 0.9,
source: "llm",
}),
derived: JSON.stringify({
phase: { kind: "photo-pending" },
intent: { kind: "status-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 0,
}),
classified: JSON.stringify({
intent: { kind: "status-question" },
confidence: 0.9,
source: "regex",
}),
validation: JSON.stringify({
status: "pass",
text: "사진 인증만 남았어요.",
reasons: [],
}),
llmJudge: JSON.stringify({
skipped: false,
model: "judge",
endpoint: "local",
provider: "test",
scores: {
friendliness: 4,
accuracy: 5,
concision: 4,
scope: 5,
overall: 4.5,
},
rationale: "ok",
raw: "{}",
}),
tier: "0",
latencyMs: "123",
evaluatedAt: "2026-05-01T00:00:00.000Z",
},
};
}
describe("archiveEvalResults", () => {
it("archives stream entries idempotently and advances the cursor", async () => {
const garnet = new MemoryGarnet([streamEntry("1-0")]);
const connection = new MemoryConnection();
const first = await archiveEvalResults(garnet, connection, {
sinceId: "0-0",
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
});
const second = await archiveEvalResults(garnet, connection, {
sinceId: "0-0",
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
});
expect(first).toEqual({ lastArchivedId: "1-0", archivedCount: 1 });
expect(second).toEqual({ lastArchivedId: "1-0", archivedCount: 0 });
expect(garnet.strings.get(EVAL_ARCHIVE_CURSOR_KEY)).toBe("1-0");
expect(connection.collectionInstance.docs.get("1-0")).toMatchObject({
streamId: "1-0",
latencyMs: 123,
tier: 0,
});
expect(connection.collectionInstance.indexes).toContainEqual({ streamId: 1 });
});
});