Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,163 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { StreamEntry } from "@common/garnet";
|
||||
import {
|
||||
archiveEvalResults,
|
||||
EVAL_ARCHIVE_CURSOR_KEY,
|
||||
EVAL_ARCHIVE_COLLECTION,
|
||||
type EvalArchiveEntry,
|
||||
} from "./archive.js";
|
||||
import { EVAL_RESULTS_STREAM_KEY } from "./aggregator.js";
|
||||
|
||||
class MemoryGarnet {
|
||||
readonly strings = new Map<string, string>();
|
||||
|
||||
constructor(private readonly entries: StreamEntry[]) {}
|
||||
|
||||
readStreamRange(
|
||||
key: string,
|
||||
start: string,
|
||||
_end: string,
|
||||
options?: { count?: number | undefined },
|
||||
): Promise<StreamEntry[]> {
|
||||
expect(key).toBe(EVAL_RESULTS_STREAM_KEY);
|
||||
const filtered = start.startsWith("(")
|
||||
? this.entries.filter((entry) => entry.id > start.slice(1))
|
||||
: this.entries;
|
||||
return Promise.resolve(filtered.slice(0, options?.count));
|
||||
}
|
||||
|
||||
setString(key: string, value: string): Promise<void> {
|
||||
this.strings.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryCollection {
|
||||
readonly docs = new Map<string, EvalArchiveEntry>();
|
||||
readonly indexes: Array<Record<string, unknown>> = [];
|
||||
|
||||
bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> {
|
||||
let upsertedCount = 0;
|
||||
for (const operation of operations) {
|
||||
const updateOne = (operation as {
|
||||
updateOne: {
|
||||
filter: { streamId: string };
|
||||
update: { $setOnInsert: EvalArchiveEntry };
|
||||
};
|
||||
}).updateOne;
|
||||
const streamId = updateOne.filter.streamId;
|
||||
if (!this.docs.has(streamId)) {
|
||||
this.docs.set(streamId, updateOne.update.$setOnInsert);
|
||||
upsertedCount += 1;
|
||||
}
|
||||
}
|
||||
return Promise.resolve({ upsertedCount });
|
||||
}
|
||||
|
||||
createIndex(keys: Record<string, unknown>): Promise<string> {
|
||||
this.indexes.push(keys);
|
||||
return Promise.resolve("idx");
|
||||
}
|
||||
|
||||
find(): never {
|
||||
throw new Error("not used");
|
||||
}
|
||||
}
|
||||
|
||||
class MemoryConnection {
|
||||
readonly collectionInstance = new MemoryCollection();
|
||||
|
||||
collection(name: string): MemoryCollection {
|
||||
expect(name).toBe(EVAL_ARCHIVE_COLLECTION);
|
||||
return this.collectionInstance;
|
||||
}
|
||||
}
|
||||
|
||||
function streamEntry(id: string): StreamEntry {
|
||||
return {
|
||||
id,
|
||||
message: {
|
||||
input: JSON.stringify({
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
userMessage: "상태 알려줘",
|
||||
},
|
||||
}),
|
||||
output: JSON.stringify({
|
||||
requestId: "req-1",
|
||||
text: "사진 인증만 남았어요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
}),
|
||||
derived: JSON.stringify({
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
}),
|
||||
classified: JSON.stringify({
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
}),
|
||||
validation: JSON.stringify({
|
||||
status: "pass",
|
||||
text: "사진 인증만 남았어요.",
|
||||
reasons: [],
|
||||
}),
|
||||
llmJudge: JSON.stringify({
|
||||
skipped: false,
|
||||
model: "judge",
|
||||
endpoint: "local",
|
||||
provider: "test",
|
||||
scores: {
|
||||
friendliness: 4,
|
||||
accuracy: 5,
|
||||
concision: 4,
|
||||
scope: 5,
|
||||
overall: 4.5,
|
||||
},
|
||||
rationale: "ok",
|
||||
raw: "{}",
|
||||
}),
|
||||
tier: "0",
|
||||
latencyMs: "123",
|
||||
evaluatedAt: "2026-05-01T00:00:00.000Z",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("archiveEvalResults", () => {
|
||||
it("archives stream entries idempotently and advances the cursor", async () => {
|
||||
const garnet = new MemoryGarnet([streamEntry("1-0")]);
|
||||
const connection = new MemoryConnection();
|
||||
|
||||
const first = await archiveEvalResults(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
const second = await archiveEvalResults(garnet, connection, {
|
||||
sinceId: "0-0",
|
||||
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(first).toEqual({ lastArchivedId: "1-0", archivedCount: 1 });
|
||||
expect(second).toEqual({ lastArchivedId: "1-0", archivedCount: 0 });
|
||||
expect(garnet.strings.get(EVAL_ARCHIVE_CURSOR_KEY)).toBe("1-0");
|
||||
expect(connection.collectionInstance.docs.get("1-0")).toMatchObject({
|
||||
streamId: "1-0",
|
||||
latencyMs: 123,
|
||||
tier: 0,
|
||||
});
|
||||
expect(connection.collectionInstance.indexes).toContainEqual({ streamId: 1 });
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user