Files
lolympus-agent/src/eval/capture.ts
T
2026-05-12 08:16:31 -07:00

135 lines
4.1 KiB
TypeScript

import { mkdir, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { initMongoDB, closeMongoDB, getMongoConnection } from "@common/mongodb";
import { getSecret } from "@common/secret-manager";
import { getLogger } from "@obs/logger";
import {
anonymizeIdentifier,
anonymizeNickname,
anonymizeRelativeTimestamp,
anonymizeText,
} from "./anonymize.js";
interface CaptureOptions {
limit?: number | undefined;
outputPath?: string | undefined;
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function extractMessageText(doc: Record<string, unknown>): string | null {
const data = asRecord(doc["data"]);
const body = asRecord(data["body"]);
const chatLog = asRecord(body["chatLog"]);
const text =
(typeof chatLog["message"] === "string" ? chatLog["message"] : null) ??
(typeof body["message"] === "string" ? body["message"] : null);
return text?.trim() || null;
}
function extractNickname(doc: Record<string, unknown>): string | undefined {
const data = asRecord(doc["data"]);
const body = asRecord(data["body"]);
return typeof body["authorNickname"] === "string"
? body["authorNickname"]
: undefined;
}
export async function captureGoldenSetScaffold(
options?: CaptureOptions,
): Promise<string> {
const logger = getLogger();
const connectionString =
getSecret("MONGODB_CONNECTION_STRING") ??
process.env["MONGODB_CONNECTION_STRING"];
if (!connectionString) {
throw new Error(
"Missing MONGODB_CONNECTION_STRING for eval capture. Ensure IMDS exposes Infisical client credentials and Infisical contains MONGODB_CONNECTION_STRING, or set it directly in the environment.",
);
}
await initMongoDB(connectionString);
try {
const connection = getMongoConnection();
const collection = connection.collection("events");
const limit = options?.limit ?? 30;
const docs = (await collection
.find({ category: "message.received" })
.sort({ timestamp: -1 })
.limit(limit)
.toArray()) as Array<Record<string, unknown>>;
const baseTimestamp = (() => {
const lastDoc = docs[docs.length - 1];
const raw = lastDoc?.["timestamp"];
const parsed =
raw instanceof Date ? raw : new Date(typeof raw === "string" ? raw : Date.now());
return parsed;
})();
const lines = docs.map((doc) => {
const rawTimestamp = doc["timestamp"];
const messageText = extractMessageText(doc) ?? "";
const chatroomId = anonymizeIdentifier(
"room",
typeof doc["chatroomId"] === "string" ? doc["chatroomId"] : undefined,
);
const userId = anonymizeIdentifier(
"user",
typeof doc["senderId"] === "string" ? doc["senderId"] : undefined,
);
const nickname = anonymizeNickname(extractNickname(doc));
return JSON.stringify({
input: {
request: {
requestId: `capture_${Math.random().toString(36).slice(2, 10)}`,
skill: "onboarding",
chatroomId: chatroomId,
userId: userId,
nickname,
trigger: "user-message",
userMessage: anonymizeText(messageText),
capturedAt: anonymizeRelativeTimestamp(
rawTimestamp instanceof Date
? rawTimestamp
: typeof rawTimestamp === "string"
? rawTimestamp
: baseTimestamp,
baseTimestamp,
),
},
roomContext: [],
session: null,
},
expected: {
intent: "",
mustIncludeAny: [],
mustNotInclude: [],
responseMode: "",
},
});
});
const outputPath = resolve(
process.cwd(),
options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl",
);
await mkdir(dirname(outputPath), { recursive: true });
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
logger.info({ count: lines.length, outputPath }, "[Eval] Capture scaffold written");
return outputPath;
} finally {
await closeMongoDB();
}
}