import { mkdir, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { initMongoDB, closeMongoDB, getMongoConnection } from "@common/mongodb"; import { getSecret } from "@common/secret-manager"; import { getLogger } from "@obs/logger"; import { anonymizeIdentifier, anonymizeNickname, anonymizeRelativeTimestamp, anonymizeText, } from "./anonymize.js"; interface CaptureOptions { limit?: number | undefined; outputPath?: string | undefined; } function asRecord(value: unknown): Record { return typeof value === "object" && value !== null ? (value as Record) : {}; } function extractMessageText(doc: Record): string | null { const data = asRecord(doc["data"]); const body = asRecord(data["body"]); const chatLog = asRecord(body["chatLog"]); const text = (typeof chatLog["message"] === "string" ? chatLog["message"] : null) ?? (typeof body["message"] === "string" ? body["message"] : null); return text?.trim() || null; } function extractNickname(doc: Record): string | undefined { const data = asRecord(doc["data"]); const body = asRecord(data["body"]); return typeof body["authorNickname"] === "string" ? body["authorNickname"] : undefined; } export async function captureGoldenSetScaffold( options?: CaptureOptions, ): Promise { const logger = getLogger(); const connectionString = getSecret("MONGODB_CONNECTION_STRING") ?? process.env["MONGODB_CONNECTION_STRING"]; if (!connectionString) { throw new Error( "Missing MONGODB_CONNECTION_STRING for eval capture. Ensure IMDS exposes Infisical client credentials and Infisical contains MONGODB_CONNECTION_STRING, or set it directly in the environment.", ); } await initMongoDB(connectionString); try { const connection = getMongoConnection(); const collection = connection.collection("events"); const limit = options?.limit ?? 30; const docs = (await collection .find({ category: "message.received" }) .sort({ timestamp: -1 }) .limit(limit) .toArray()) as Array>; const baseTimestamp = (() => { const lastDoc = docs[docs.length - 1]; const raw = lastDoc?.["timestamp"]; const parsed = raw instanceof Date ? raw : new Date(typeof raw === "string" ? raw : Date.now()); return parsed; })(); const lines = docs.map((doc) => { const rawTimestamp = doc["timestamp"]; const messageText = extractMessageText(doc) ?? ""; const chatroomId = anonymizeIdentifier( "room", typeof doc["chatroomId"] === "string" ? doc["chatroomId"] : undefined, ); const userId = anonymizeIdentifier( "user", typeof doc["senderId"] === "string" ? doc["senderId"] : undefined, ); const nickname = anonymizeNickname(extractNickname(doc)); return JSON.stringify({ input: { request: { requestId: `capture_${Math.random().toString(36).slice(2, 10)}`, skill: "onboarding", chatroomId: chatroomId, userId: userId, nickname, trigger: "user-message", userMessage: anonymizeText(messageText), capturedAt: anonymizeRelativeTimestamp( rawTimestamp instanceof Date ? rawTimestamp : typeof rawTimestamp === "string" ? rawTimestamp : baseTimestamp, baseTimestamp, ), }, roomContext: [], session: null, }, expected: { intent: "", mustIncludeAny: [], mustNotInclude: [], responseMode: "", }, }); }); const outputPath = resolve( process.cwd(), options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl", ); await mkdir(dirname(outputPath), { recursive: true }); await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8"); logger.info({ count: lines.length, outputPath }, "[Eval] Capture scaffold written"); return outputPath; } finally { await closeMongoDB(); } }