init
This commit is contained in:
commit
7fa293c9e8
59 files changed
+15526
No files matched your search
@@ -0,0 +1,132 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import { initMongoDB, closeMongoDB, getMongoConnection } from "@common/mongodb";
|
||||
import { getSecret } from "@common/secret-manager";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import {
|
||||
anonymizeIdentifier,
|
||||
anonymizeNickname,
|
||||
anonymizeRelativeTimestamp,
|
||||
anonymizeText,
|
||||
} from "./anonymize.js";
|
||||
|
||||
interface CaptureOptions {
|
||||
limit?: number | undefined;
|
||||
outputPath?: string | undefined;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function extractMessageText(doc: Record<string, unknown>): string | null {
|
||||
const data = asRecord(doc["data"]);
|
||||
const body = asRecord(data["body"]);
|
||||
const chatLog = asRecord(body["chatLog"]);
|
||||
const text =
|
||||
(typeof chatLog["message"] === "string" ? chatLog["message"] : null) ??
|
||||
(typeof body["message"] === "string" ? body["message"] : null);
|
||||
|
||||
return text?.trim() || null;
|
||||
}
|
||||
|
||||
function extractNickname(doc: Record<string, unknown>): string | undefined {
|
||||
const data = asRecord(doc["data"]);
|
||||
const body = asRecord(data["body"]);
|
||||
return typeof body["authorNickname"] === "string"
|
||||
? body["authorNickname"]
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export async function captureGoldenSetScaffold(
|
||||
options?: CaptureOptions,
|
||||
): Promise<string> {
|
||||
const logger = getLogger();
|
||||
const connectionString =
|
||||
getSecret("MONGODB_CONNECTION_STRING") ??
|
||||
process.env["MONGODB_CONNECTION_STRING"];
|
||||
|
||||
if (!connectionString) {
|
||||
throw new Error("Missing MONGODB_CONNECTION_STRING for eval capture");
|
||||
}
|
||||
|
||||
await initMongoDB(connectionString);
|
||||
|
||||
try {
|
||||
const connection = getMongoConnection();
|
||||
const collection = connection.collection("events");
|
||||
const limit = options?.limit ?? 30;
|
||||
|
||||
const docs = (await collection
|
||||
.find({ category: "message.received" })
|
||||
.sort({ timestamp: -1 })
|
||||
.limit(limit)
|
||||
.toArray()) as Array<Record<string, unknown>>;
|
||||
|
||||
const baseTimestamp = (() => {
|
||||
const lastDoc = docs[docs.length - 1];
|
||||
const raw = lastDoc?.["timestamp"];
|
||||
const parsed =
|
||||
raw instanceof Date ? raw : new Date(typeof raw === "string" ? raw : Date.now());
|
||||
return parsed;
|
||||
})();
|
||||
|
||||
const lines = docs.map((doc) => {
|
||||
const rawTimestamp = doc["timestamp"];
|
||||
const messageText = extractMessageText(doc) ?? "";
|
||||
const chatroomId = anonymizeIdentifier(
|
||||
"room",
|
||||
typeof doc["chatroomId"] === "string" ? doc["chatroomId"] : undefined,
|
||||
);
|
||||
const userId = anonymizeIdentifier(
|
||||
"user",
|
||||
typeof doc["senderId"] === "string" ? doc["senderId"] : undefined,
|
||||
);
|
||||
const nickname = anonymizeNickname(extractNickname(doc));
|
||||
|
||||
return JSON.stringify({
|
||||
input: {
|
||||
request: {
|
||||
requestId: `capture_${Math.random().toString(36).slice(2, 10)}`,
|
||||
skill: "onboarding",
|
||||
chatroomId: chatroomId,
|
||||
userId: userId,
|
||||
nickname,
|
||||
trigger: "user-message",
|
||||
userMessage: anonymizeText(messageText),
|
||||
capturedAt: anonymizeRelativeTimestamp(
|
||||
rawTimestamp instanceof Date
|
||||
? rawTimestamp
|
||||
: typeof rawTimestamp === "string"
|
||||
? rawTimestamp
|
||||
: baseTimestamp,
|
||||
baseTimestamp,
|
||||
),
|
||||
},
|
||||
roomContext: [],
|
||||
session: null,
|
||||
},
|
||||
expected: {
|
||||
intent: "",
|
||||
mustIncludeAny: [],
|
||||
mustNotInclude: [],
|
||||
responseMode: "",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
const outputPath = resolve(
|
||||
process.cwd(),
|
||||
options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl",
|
||||
);
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
|
||||
logger.info({ count: lines.length, outputPath }, "[Eval] Capture scaffold written");
|
||||
return outputPath;
|
||||
} finally {
|
||||
await closeMongoDB();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user