164 lines
4.5 KiB
TypeScript
164 lines
4.5 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import type { StreamEntry } from "@common/garnet";
|
|
import {
|
|
archiveEvalResults,
|
|
EVAL_ARCHIVE_CURSOR_KEY,
|
|
EVAL_ARCHIVE_COLLECTION,
|
|
type EvalArchiveEntry,
|
|
} from "./archive.js";
|
|
import { EVAL_RESULTS_STREAM_KEY } from "./aggregator.js";
|
|
|
|
class MemoryGarnet {
|
|
readonly strings = new Map<string, string>();
|
|
|
|
constructor(private readonly entries: StreamEntry[]) {}
|
|
|
|
readStreamRange(
|
|
key: string,
|
|
start: string,
|
|
_end: string,
|
|
options?: { count?: number | undefined },
|
|
): Promise<StreamEntry[]> {
|
|
expect(key).toBe(EVAL_RESULTS_STREAM_KEY);
|
|
const filtered = start.startsWith("(")
|
|
? this.entries.filter((entry) => entry.id > start.slice(1))
|
|
: this.entries;
|
|
return Promise.resolve(filtered.slice(0, options?.count));
|
|
}
|
|
|
|
setString(key: string, value: string): Promise<void> {
|
|
this.strings.set(key, value);
|
|
return Promise.resolve();
|
|
}
|
|
}
|
|
|
|
class MemoryCollection {
|
|
readonly docs = new Map<string, EvalArchiveEntry>();
|
|
readonly indexes: Array<Record<string, unknown>> = [];
|
|
|
|
bulkWrite(operations: unknown[]): Promise<{ upsertedCount: number }> {
|
|
let upsertedCount = 0;
|
|
for (const operation of operations) {
|
|
const updateOne = (operation as {
|
|
updateOne: {
|
|
filter: { streamId: string };
|
|
update: { $setOnInsert: EvalArchiveEntry };
|
|
};
|
|
}).updateOne;
|
|
const streamId = updateOne.filter.streamId;
|
|
if (!this.docs.has(streamId)) {
|
|
this.docs.set(streamId, updateOne.update.$setOnInsert);
|
|
upsertedCount += 1;
|
|
}
|
|
}
|
|
return Promise.resolve({ upsertedCount });
|
|
}
|
|
|
|
createIndex(keys: Record<string, unknown>): Promise<string> {
|
|
this.indexes.push(keys);
|
|
return Promise.resolve("idx");
|
|
}
|
|
|
|
find(): never {
|
|
throw new Error("not used");
|
|
}
|
|
}
|
|
|
|
class MemoryConnection {
|
|
readonly collectionInstance = new MemoryCollection();
|
|
|
|
collection(name: string): MemoryCollection {
|
|
expect(name).toBe(EVAL_ARCHIVE_COLLECTION);
|
|
return this.collectionInstance;
|
|
}
|
|
}
|
|
|
|
function streamEntry(id: string): StreamEntry {
|
|
return {
|
|
id,
|
|
message: {
|
|
input: JSON.stringify({
|
|
request: {
|
|
requestId: "req-1",
|
|
skill: "onboarding",
|
|
chatroomId: "room-1",
|
|
userId: "user-1",
|
|
nickname: "닉",
|
|
trigger: "user-message",
|
|
userMessage: "상태 알려줘",
|
|
},
|
|
}),
|
|
output: JSON.stringify({
|
|
requestId: "req-1",
|
|
text: "사진 인증만 남았어요.",
|
|
confidence: 0.9,
|
|
source: "llm",
|
|
}),
|
|
derived: JSON.stringify({
|
|
phase: { kind: "photo-pending" },
|
|
intent: { kind: "status-question" },
|
|
shouldRespond: true,
|
|
responseMode: "llm",
|
|
suggestedMaxTokens: 128,
|
|
allowShortResponse: true,
|
|
tier: 0,
|
|
}),
|
|
classified: JSON.stringify({
|
|
intent: { kind: "status-question" },
|
|
confidence: 0.9,
|
|
source: "regex",
|
|
}),
|
|
validation: JSON.stringify({
|
|
status: "pass",
|
|
text: "사진 인증만 남았어요.",
|
|
reasons: [],
|
|
}),
|
|
llmJudge: JSON.stringify({
|
|
skipped: false,
|
|
model: "judge",
|
|
endpoint: "local",
|
|
provider: "test",
|
|
scores: {
|
|
friendliness: 4,
|
|
accuracy: 5,
|
|
concision: 4,
|
|
scope: 5,
|
|
overall: 4.5,
|
|
},
|
|
rationale: "ok",
|
|
raw: "{}",
|
|
}),
|
|
tier: "0",
|
|
latencyMs: "123",
|
|
evaluatedAt: "2026-05-01T00:00:00.000Z",
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("archiveEvalResults", () => {
|
|
it("archives stream entries idempotently and advances the cursor", async () => {
|
|
const garnet = new MemoryGarnet([streamEntry("1-0")]);
|
|
const connection = new MemoryConnection();
|
|
|
|
const first = await archiveEvalResults(garnet, connection, {
|
|
sinceId: "0-0",
|
|
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
|
});
|
|
const second = await archiveEvalResults(garnet, connection, {
|
|
sinceId: "0-0",
|
|
nowMs: Date.parse("2026-05-01T00:00:00.000Z"),
|
|
});
|
|
|
|
expect(first).toEqual({ lastArchivedId: "1-0", archivedCount: 1 });
|
|
expect(second).toEqual({ lastArchivedId: "1-0", archivedCount: 0 });
|
|
expect(garnet.strings.get(EVAL_ARCHIVE_CURSOR_KEY)).toBe("1-0");
|
|
expect(connection.collectionInstance.docs.get("1-0")).toMatchObject({
|
|
streamId: "1-0",
|
|
latencyMs: 123,
|
|
tier: 0,
|
|
});
|
|
expect(connection.collectionInstance.indexes).toContainEqual({ streamId: 1 });
|
|
});
|
|
});
|