Initialize onboarding agent scaffold

This commit is contained in:
dahyeon committed 2026-04-16 13:48:53 -07:00
commit 4294f3c92e
57 files changed
+14388

No files matched your search

+53
View File
@@ -0,0 +1,53 @@
import { createHash } from "node:crypto";
const EVAL_ANON_SALT = "lolympus-agent-v1" as const;
function shortHash(value: string): string {
return createHash("sha256")
.update(`${EVAL_ANON_SALT}:${value}`)
.digest("hex")
.slice(0, 8);
}
export function anonymizeIdentifier(
prefix: string,
value: string | undefined,
): string | undefined {
if (!value?.trim()) {
return undefined;
}
return `${prefix}_${shortHash(value)}`;
}
export function anonymizeNickname(value: string | undefined): string | undefined {
if (!value?.trim()) {
return undefined;
}
return `<nick:${shortHash(value)}>`;
}
export function anonymizeText(value: string): string {
return value
.replace(/01[016789][- ]?\d{3,4}[- ]?\d{4}/gu, "[REDACTED:phone]")
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/giu, "[REDACTED:email]")
.replace(/\b(?:\d[ -]?){13,19}\b/gu, "[REDACTED:card]")
.replace(/https?:\/\/open\.kakao\.com\/o\/[^\s)]+/giu, "[REDACTED:kakao-link]")
.replace(/https?:\/\/discord\.gg\/[^\s)]+/giu, "[REDACTED:discord-link]")
.replace(/discord\.gg\/[^\s)]+/giu, "[REDACTED:discord-link]");
}
export function anonymizeRelativeTimestamp(
value: Date | string,
baseTimestamp: Date,
): string {
const timestamp =
value instanceof Date ? value.getTime() : Date.parse(value);
if (!Number.isFinite(timestamp)) {
return "+0s";
}
const seconds = Math.round((timestamp - baseTimestamp.getTime()) / 1000);
return `${seconds >= 0 ? "+" : ""}${seconds}s`;
}
+132
View File
@@ -0,0 +1,132 @@
import { mkdir, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { initMongoDB, closeMongoDB, getMongoConnection } from "@common/mongodb";
import { getSecret } from "@common/secret-manager";
import { getLogger } from "@obs/logger";
import {
anonymizeIdentifier,
anonymizeNickname,
anonymizeRelativeTimestamp,
anonymizeText,
} from "./anonymize.js";
interface CaptureOptions {
limit?: number | undefined;
outputPath?: string | undefined;
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function extractMessageText(doc: Record<string, unknown>): string | null {
const data = asRecord(doc["data"]);
const body = asRecord(data["body"]);
const chatLog = asRecord(body["chatLog"]);
const text =
(typeof chatLog["message"] === "string" ? chatLog["message"] : null) ??
(typeof body["message"] === "string" ? body["message"] : null);
return text?.trim() || null;
}
function extractNickname(doc: Record<string, unknown>): string | undefined {
const data = asRecord(doc["data"]);
const body = asRecord(data["body"]);
return typeof body["authorNickname"] === "string"
? body["authorNickname"]
: undefined;
}
export async function captureGoldenSetScaffold(
options?: CaptureOptions,
): Promise<string> {
const logger = getLogger();
const connectionString =
getSecret("MONGODB_CONNECTION_STRING") ??
process.env["MONGODB_CONNECTION_STRING"];
if (!connectionString) {
throw new Error("Missing MONGODB_CONNECTION_STRING for eval capture");
}
await initMongoDB(connectionString);
try {
const connection = getMongoConnection();
const collection = connection.collection("events");
const limit = options?.limit ?? 30;
const docs = (await collection
.find({ category: "message.received" })
.sort({ timestamp: -1 })
.limit(limit)
.toArray()) as Array<Record<string, unknown>>;
const baseTimestamp = (() => {
const lastDoc = docs[docs.length - 1];
const raw = lastDoc?.["timestamp"];
const parsed =
raw instanceof Date ? raw : new Date(typeof raw === "string" ? raw : Date.now());
return parsed;
})();
const lines = docs.map((doc) => {
const rawTimestamp = doc["timestamp"];
const messageText = extractMessageText(doc) ?? "";
const chatroomId = anonymizeIdentifier(
"room",
typeof doc["chatroomId"] === "string" ? doc["chatroomId"] : undefined,
);
const userId = anonymizeIdentifier(
"user",
typeof doc["senderId"] === "string" ? doc["senderId"] : undefined,
);
const nickname = anonymizeNickname(extractNickname(doc));
return JSON.stringify({
input: {
request: {
requestId: `capture_${Math.random().toString(36).slice(2, 10)}`,
skill: "onboarding",
chatroomId: chatroomId,
userId: userId,
nickname,
trigger: "user-message",
userMessage: anonymizeText(messageText),
capturedAt: anonymizeRelativeTimestamp(
rawTimestamp instanceof Date
? rawTimestamp
: typeof rawTimestamp === "string"
? rawTimestamp
: baseTimestamp,
baseTimestamp,
),
},
roomContext: [],
session: null,
},
expected: {
intent: "",
mustIncludeAny: [],
mustNotInclude: [],
responseMode: "",
},
});
});
const outputPath = resolve(
process.cwd(),
options?.outputPath ?? "src/eval/golden/onboarding.capture.jsonl",
);
await mkdir(dirname(outputPath), { recursive: true });
await writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
logger.info({ count: lines.length, outputPath }, "[Eval] Capture scaffold written");
return outputPath;
} finally {
await closeMongoDB();
}
}
+38
View File
@@ -0,0 +1,38 @@
function printPhaseNotice(command: string): void {
console.info(
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
);
}
function parsePositiveInteger(value: string | undefined): number | undefined {
if (!value?.trim()) {
return undefined;
}
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
async function main(): Promise<void> {
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
loadEnvironmentFromNodeEnv();
const { getLogger, initLogger } = await import("@obs/logger");
initLogger();
const { captureGoldenSetScaffold } = await import("./capture.js");
const [command = "run", arg1, arg2] = process.argv.slice(2);
if (command === "capture") {
const outputPath = await captureGoldenSetScaffold({
limit: parsePositiveInteger(arg1),
outputPath: arg2,
});
getLogger().info({ outputPath }, '[eval] "capture" completed');
return;
}
printPhaseNotice(command);
}
void main();