135 lines
4.1 KiB
TypeScript
135 lines
4.1 KiB
TypeScript
import { mkdir, writeFile } from "node:fs/promises";
|
|
import { dirname, resolve } from "node:path";
|
|
|
|
import { initMongoDB, closeMongoDB, getMongoConnection } from "@common/mongodb";
|
|
import { getSecret } from "@common/secret-manager";
|
|
import { getLogger } from "@obs/logger";
|
|
import {
|
|
anonymizeIdentifier,
|
|
anonymizeNickname,
|
|
anonymizeRelativeTimestamp,
|
|
anonymizeText,
|
|
} from "./anonymize.js";
|
|
|
|
interface CaptureOptions {
|
|
limit?: number | undefined;
|
|
outputPath?: string | undefined;
|
|
}
|
|
|
|
function asRecord(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function extractMessageText(doc: Record<string, unknown>): string | null {
|
|
const data = asRecord(doc["data"]);
|
|
const body = asRecord(data["body"]);
|
|
const chatLog = asRecord(body["chatLog"]);
|
|
const text =
|
|
(typeof chatLog["message"] === "string" ? chatLog["message"] : null) ??
|
|
(typeof body["message"] === "string" ? body["message"] : null);
|
|
|
|
return text?.trim() || null;
|
|
}
|
|
|
|
function extractNickname(doc: Record<string, unknown>): string | undefined {
|
|
const data = asRecord(doc["data"]);
|
|
const body = asRecord(data["body"]);
|
|
return typeof body["authorNickname"] === "string"
|
|
? body["authorNickname"]
|
|
: undefined;
|
|
}
|
|
|
|
export async function captureGoldenSetScaffold(
|
|
options?: CaptureOptions,
|
|
): Promise<string> {
|
|
const logger = getLogger();
|
|
const connectionString =
|
|
getSecret("MONGODB_CONNECTION_STRING") ??
|
|
process.env["MONGODB_CONNECTION_STRING"];
|
|
|
|
if (!connectionString) {
|
|
throw new Error(
|
|
"Missing MONGODB_CONNECTION_STRING for eval capture. Ensure IMDS exposes Infisical client credentials and Infisical contains MONGODB_CONNECTION_STRING, or set it directly in the environment.",
|
|
);
|
|
}
|
|
|
|
await initMongoDB(connectionString);
|
|
|
|
try {
|
|
const connection = getMongoConnection();
|
|
const collection = connection.collection("events");
|
|
const limit = options?.limit ?? 30;
|
|
|
|
const docs = (await collection
|
|
.find({ category: "message.received" })
|
|
.sort({ timestamp: -1 })
|
|
.limit(limit)
|
|
.toArray()) as Array<Record<string, unknown>>;
|
|
|
|
const baseTimestamp = (() => {
|
|
const lastDoc = docs[docs.length - 1];
|
|
const raw = lastDoc?.["timestamp"];
|
|
const parsed =
|
|
raw instanceof Date ? raw : new Date(typeof raw === "string" ? raw : Date.now());
|
|
return parsed;
|
|
})();
|
|
|
|
const lines = docs.map((doc) => {
|
|
const rawTimestamp = doc["timestamp"];
|
|
const messageText = extractMessageText(doc) ?? "";
|
|
const chatroomId = anonymizeIdentifier(
|
|
"room",
|
|
typeof doc["chatroomId"] === "string" ? doc["chatroomId"] : undefined,
|
|
);
|
|
const userId = anonymizeIdentifier(
|
|
"user",
|
|
typeof doc["senderId"] === "string" ? doc["senderId"] : undefined,
|
|
);
|
|
const nickname = anonymizeNickname(extractNickname(doc));
|
|
|
|
return JSON.stringify({
|
|
input: {
|
|
request: {
|
|
requestId: `capture_${Math.random().toString(36).slice(2, 10)}`,
|
|
skill: "onboarding",
|
|
chatroomId: chatroomId,
|
|
userId: userId,
|
|
nickname,
|
|
trigger: "user-message",
|
|
userMessage: anonymizeText(messageText),
|
|
capturedAt: anonymizeRelativeTimestamp(
|
|
rawTimestamp instanceof Date
|
|
? rawTimestamp
|
|
: typeof rawTimestamp === "string"
|
|
? rawTimestamp
|
|
: baseTimestamp,
|
|
baseTimestamp,
|
|
),
|
|
},
|
|
roomContext: [],
|
|
session: null,
|
|
},
|
|
expected: {
|
|
intent: "",
|
|
mustIncludeAny: [],
|
|
mustNotInclude: [],
|
|
responseMode: "",
|
|
},
|
|
});
|
|
});
|
|
|
|
const outputPath = resolve(
|
|
process.cwd(),
|
|
options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl",
|
|
);
|
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
|
|
logger.info({ count: lines.length, outputPath }, "[Eval] Capture scaffold written");
|
|
return outputPath;
|
|
} finally {
|
|
await closeMongoDB();
|
|
}
|
|
}
|