Initialize onboarding agent scaffold
This commit is contained in:
commit
4294f3c92e
57 files changed
+14388
No files matched your search
@@ -0,0 +1,53 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
const EVAL_ANON_SALT = "lolympus-agent-v1" as const;
|
||||
|
||||
function shortHash(value: string): string {
|
||||
return createHash("sha256")
|
||||
.update(`${EVAL_ANON_SALT}:${value}`)
|
||||
.digest("hex")
|
||||
.slice(0, 8);
|
||||
}
|
||||
|
||||
export function anonymizeIdentifier(
|
||||
prefix: string,
|
||||
value: string | undefined,
|
||||
): string | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return `${prefix}_${shortHash(value)}`;
|
||||
}
|
||||
|
||||
export function anonymizeNickname(value: string | undefined): string | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return `<nick:${shortHash(value)}>`;
|
||||
}
|
||||
|
||||
export function anonymizeText(value: string): string {
|
||||
return value
|
||||
.replace(/01[016789][- ]?\d{3,4}[- ]?\d{4}/gu, "[REDACTED:phone]")
|
||||
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/giu, "[REDACTED:email]")
|
||||
.replace(/\b(?:\d[ -]?){13,19}\b/gu, "[REDACTED:card]")
|
||||
.replace(/https?:\/\/open\.kakao\.com\/o\/[^\s)]+/giu, "[REDACTED:kakao-link]")
|
||||
.replace(/https?:\/\/discord\.gg\/[^\s)]+/giu, "[REDACTED:discord-link]")
|
||||
.replace(/discord\.gg\/[^\s)]+/giu, "[REDACTED:discord-link]");
|
||||
}
|
||||
|
||||
export function anonymizeRelativeTimestamp(
|
||||
value: Date | string,
|
||||
baseTimestamp: Date,
|
||||
): string {
|
||||
const timestamp =
|
||||
value instanceof Date ? value.getTime() : Date.parse(value);
|
||||
if (!Number.isFinite(timestamp)) {
|
||||
return "+0s";
|
||||
}
|
||||
|
||||
const seconds = Math.round((timestamp - baseTimestamp.getTime()) / 1000);
|
||||
return `${seconds >= 0 ? "+" : ""}${seconds}s`;
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import { initMongoDB, closeMongoDB, getMongoConnection } from "@common/mongodb";
|
||||
import { getSecret } from "@common/secret-manager";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import {
|
||||
anonymizeIdentifier,
|
||||
anonymizeNickname,
|
||||
anonymizeRelativeTimestamp,
|
||||
anonymizeText,
|
||||
} from "./anonymize.js";
|
||||
|
||||
interface CaptureOptions {
|
||||
limit?: number | undefined;
|
||||
outputPath?: string | undefined;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function extractMessageText(doc: Record<string, unknown>): string | null {
|
||||
const data = asRecord(doc["data"]);
|
||||
const body = asRecord(data["body"]);
|
||||
const chatLog = asRecord(body["chatLog"]);
|
||||
const text =
|
||||
(typeof chatLog["message"] === "string" ? chatLog["message"] : null) ??
|
||||
(typeof body["message"] === "string" ? body["message"] : null);
|
||||
|
||||
return text?.trim() || null;
|
||||
}
|
||||
|
||||
function extractNickname(doc: Record<string, unknown>): string | undefined {
|
||||
const data = asRecord(doc["data"]);
|
||||
const body = asRecord(data["body"]);
|
||||
return typeof body["authorNickname"] === "string"
|
||||
? body["authorNickname"]
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export async function captureGoldenSetScaffold(
|
||||
options?: CaptureOptions,
|
||||
): Promise<string> {
|
||||
const logger = getLogger();
|
||||
const connectionString =
|
||||
getSecret("MONGODB_CONNECTION_STRING") ??
|
||||
process.env["MONGODB_CONNECTION_STRING"];
|
||||
|
||||
if (!connectionString) {
|
||||
throw new Error("Missing MONGODB_CONNECTION_STRING for eval capture");
|
||||
}
|
||||
|
||||
await initMongoDB(connectionString);
|
||||
|
||||
try {
|
||||
const connection = getMongoConnection();
|
||||
const collection = connection.collection("events");
|
||||
const limit = options?.limit ?? 30;
|
||||
|
||||
const docs = (await collection
|
||||
.find({ category: "message.received" })
|
||||
.sort({ timestamp: -1 })
|
||||
.limit(limit)
|
||||
.toArray()) as Array<Record<string, unknown>>;
|
||||
|
||||
const baseTimestamp = (() => {
|
||||
const lastDoc = docs[docs.length - 1];
|
||||
const raw = lastDoc?.["timestamp"];
|
||||
const parsed =
|
||||
raw instanceof Date ? raw : new Date(typeof raw === "string" ? raw : Date.now());
|
||||
return parsed;
|
||||
})();
|
||||
|
||||
const lines = docs.map((doc) => {
|
||||
const rawTimestamp = doc["timestamp"];
|
||||
const messageText = extractMessageText(doc) ?? "";
|
||||
const chatroomId = anonymizeIdentifier(
|
||||
"room",
|
||||
typeof doc["chatroomId"] === "string" ? doc["chatroomId"] : undefined,
|
||||
);
|
||||
const userId = anonymizeIdentifier(
|
||||
"user",
|
||||
typeof doc["senderId"] === "string" ? doc["senderId"] : undefined,
|
||||
);
|
||||
const nickname = anonymizeNickname(extractNickname(doc));
|
||||
|
||||
return JSON.stringify({
|
||||
input: {
|
||||
request: {
|
||||
requestId: `capture_${Math.random().toString(36).slice(2, 10)}`,
|
||||
skill: "onboarding",
|
||||
chatroomId: chatroomId,
|
||||
userId: userId,
|
||||
nickname,
|
||||
trigger: "user-message",
|
||||
userMessage: anonymizeText(messageText),
|
||||
capturedAt: anonymizeRelativeTimestamp(
|
||||
rawTimestamp instanceof Date
|
||||
? rawTimestamp
|
||||
: typeof rawTimestamp === "string"
|
||||
? rawTimestamp
|
||||
: baseTimestamp,
|
||||
baseTimestamp,
|
||||
),
|
||||
},
|
||||
roomContext: [],
|
||||
session: null,
|
||||
},
|
||||
expected: {
|
||||
intent: "",
|
||||
mustIncludeAny: [],
|
||||
mustNotInclude: [],
|
||||
responseMode: "",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
const outputPath = resolve(
|
||||
process.cwd(),
|
||||
options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl",
|
||||
);
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
|
||||
logger.info({ count: lines.length, outputPath }, "[Eval] Capture scaffold written");
|
||||
return outputPath;
|
||||
} finally {
|
||||
await closeMongoDB();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
function printPhaseNotice(command: string): void {
|
||||
console.info(
|
||||
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
|
||||
loadEnvironmentFromNodeEnv();
|
||||
|
||||
const { getLogger, initLogger } = await import("@obs/logger");
|
||||
initLogger();
|
||||
const { captureGoldenSetScaffold } = await import("./capture.js");
|
||||
|
||||
const [command = "run", arg1, arg2] = process.argv.slice(2);
|
||||
|
||||
if (command === "capture") {
|
||||
const outputPath = await captureGoldenSetScaffold({
|
||||
limit: parsePositiveInteger(arg1),
|
||||
outputPath: arg2,
|
||||
});
|
||||
getLogger().info({ outputPath }, '[eval] "capture" completed');
|
||||
return;
|
||||
}
|
||||
|
||||
printPhaseNotice(command);
|
||||
}
|
||||
|
||||
void main();
|
||||
Reference in new issue
Block a user