import { createHash } from "node:crypto"; import { mkdir, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { z } from "zod"; import type { LlmClient } from "@llm/llm-client"; import { SkillRegistry } from "@skills/skill-loader"; import { createBuiltInToolRegistry } from "@tools/builtins/index"; import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state"; import type { Blackboard, PromptContext } from "@skills/onboarding/types"; import { PromptBuilder, type PromptLayerDiff, type PromptLayerName, type PromptVariantManifest, } from "@skills/onboarding/prompts/builder"; import { getEvalTournamentMinCases, getLlmProposeMaxTokens, getLlmProposeModel, } from "../../config.js"; import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js"; import { getEvalArchiveCollection } from "../archive.js"; import type { SliceKey } from "../aggregator.js"; import { queryArchivedReviewLabels } from "../review/label-archive.js"; import type { ReviewLabel } from "../review/queue.js"; import { runG1Static, runG4Permission } from "../governance/gates.js"; import { promoteTrustTier } from "../governance/trust.js"; export interface ProposalInput { slice: SliceKey; recentLabels: ReviewLabel[]; rootCauses?: RootCauseReport[] | undefined; worstCases: EvalArchiveEntry[]; currentPromptSnapshot: string; } export interface RootCauseReport { avgScoreLoss: number; count: number; examples: string[]; rootCauseStep: string; } export interface PromptVariantProposal extends PromptVariantManifest { description: string; expectedImprovement: string; rationale: string; } export interface ToolProposal { description: string; schema: unknown; toolName: string; variantId: string; whyNeeded: string; } const promptDiffSchema = z.object({ layer: z.enum([ "context", "examples", "persona", "policy", "state", "tool-policy", ]), op: z.enum(["append", "remove", "replace"]), reason: z.string().trim().max(500).optional(), text: z.string().trim().max(4_000).optional(), }); const proposalSchema = z.object({ prompts: z .array( z.object({ description: z.string().trim().min(1).max(300), diff: z.array(promptDiffSchema).min(1).max(10), expectedImprovement: z.string().trim().min(1).max(500), rationale: z.string().trim().min(1).max(500), }), ) .max(5), tools: z .array( z.object({ description: z.string().trim().min(1).max(300), schema: z.unknown(), toolName: z.string().trim().min(1).max(80), whyNeeded: z.string().trim().min(1).max(500), }), ) .max(5) .default([]), }); function stableHash(value: unknown): string { return createHash("sha256") .update(JSON.stringify(value)) .digest("hex") .slice(0, 8); } function normalizePromptDiff(diff: z.infer): PromptLayerDiff { const normalized: PromptLayerDiff = { layer: diff.layer as PromptLayerName, op: diff.op, }; if (diff.reason !== undefined) { normalized.reason = diff.reason; } if (diff.text !== undefined) { normalized.text = diff.text; } return normalized; } function getLlmOverall(entry: EvalArchiveEntry): number | null { return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall; } function emptyBlackboard(): Blackboard { return { flaggedIssues: [], openQuestions: [], recentNicknameAttempts: [], toneMemory: {}, }; } function derivedFromSlice(slice: SliceKey): DerivedState { return { allowShortResponse: false, intent: { kind: slice.intent } as Intent, phase: { kind: slice.phase } as OnboardingPhase, responseMode: "llm", shouldRespond: true, suggestedMaxTokens: 384, tier: slice.tier, }; } function defaultPromptContext(): PromptContext { return { blackboard: emptyBlackboard(), conversationHistory: [], userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.", }; } export function buildCurrentPromptSnapshot( slice: SliceKey, archiveEntry?: EvalArchiveEntry, ): string { return PromptBuilder.forDerived( archiveEntry?.derived ?? derivedFromSlice(slice), null, defaultPromptContext(), ).buildSystemPrompt(); } function summarizeArchiveEntry(entry: EvalArchiveEntry): Record { return { streamId: entry.streamId, request: entry.request, response: entry.response.text, validation: entry.validation, llmJudge: entry.llmJudge, processJudge: entry.processJudge, process: entry.process, evaluatedAt: entry.evaluatedAt.toISOString(), }; } function round(value: number): number { return Number(value.toFixed(3)); } export function summarizeRootCauses( entries: EvalArchiveEntry[], ): RootCauseReport[] { const groups = new Map< string, { examples: Array<{ loss: number; streamId: string }>; lossSum: number } >(); for (const entry of entries) { const rootCauseStep = entry.processJudge?.rootCauseStep; const rootCauseScoreLoss = entry.processJudge?.rootCauseScoreLoss ?? 0; if (!rootCauseStep || rootCauseScoreLoss <= 0) { continue; } const group = groups.get(rootCauseStep) ?? { examples: [], lossSum: 0, }; group.examples.push({ loss: rootCauseScoreLoss, streamId: entry.streamId, }); group.lossSum += rootCauseScoreLoss; groups.set(rootCauseStep, group); } return [...groups.entries()] .map(([rootCauseStep, group]) => ({ avgScoreLoss: round(group.lossSum / group.examples.length), count: group.examples.length, examples: group.examples .sort((left, right) => right.loss - left.loss) .slice(0, 3) .map((example) => example.streamId), rootCauseStep, })) .sort( (left, right) => right.count - left.count || right.avgScoreLoss - left.avgScoreLoss || left.rootCauseStep.localeCompare(right.rootCauseStep), ) .slice(0, 5); } function buildProposalPrompt(input: ProposalInput): string { return [ "온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.", "반드시 JSON object만 출력하세요.", "prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.", "과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.", "", "[slice]", JSON.stringify(input.slice), "", "[recent_human_labels]", JSON.stringify(input.recentLabels.slice(0, 20)), "", "[root_causes]", JSON.stringify(input.rootCauses ?? []), "", "[worst_cases]", JSON.stringify(input.worstCases.slice(0, 5).map(summarizeArchiveEntry)), "", "[current_prompt_snapshot]", input.currentPromptSnapshot, "", 'JSON schema: {"prompts":[{"description":"...","diff":[{"layer":"policy","op":"append","text":"...","reason":"..."}],"rationale":"...","expectedImprovement":"..."}],"tools":[{"description":"...","toolName":"...","schema":{},"whyNeeded":"..."}]}', ].join("\n"); } export async function proposeVariants( llmClient: LlmClient, input: ProposalInput, ): Promise<{ prompts: PromptVariantProposal[]; tools: ToolProposal[] }> { if (!llmClient.hasEndpoints()) { throw new Error("Cannot propose variants without an LLM endpoint"); } const response = await llmClient.chatCompletion( [ { role: "system", content: "당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.", }, { role: "user", content: buildProposalPrompt(input), }, ], { jsonMode: true, maxTokens: getLlmProposeMaxTokens(), modelOverride: getLlmProposeModel(), temperature: 0.2, }, ); if (!response) { throw new Error("Proposal LLM call failed"); } const parsed = proposalSchema.parse(JSON.parse(response.content)); const prompts = parsed.prompts.map((proposal) => { const diff = proposal.diff.map(normalizePromptDiff); return { ...proposal, diff, variantId: stableHash({ description: proposal.description, diff, rationale: proposal.rationale, }), }; }); const tools = parsed.tools.map((proposal) => ({ ...proposal, variantId: stableHash(proposal), })); return { prompts, tools, }; } export async function buildProposalInputFromArchive( mongoConnection: MongoConnectionLike, slice: SliceKey, options?: { force?: boolean | undefined; limit?: number | undefined; minCases?: number | undefined; nowMs?: number | undefined; }, ): Promise { const limit = options?.limit ?? 50; const entries = await getEvalArchiveCollection(mongoConnection) .find({ tier: slice.tier, "derived.intent.kind": slice.intent, "derived.phase.kind": slice.phase, }) .sort({ evaluatedAt: -1 }) .limit(limit) .toArray(); const minCases = options?.minCases ?? getEvalTournamentMinCases(); if (!options?.force && entries.length < minCases) { throw new Error( `Not enough archived cases for propose: ${entries.length}/${minCases}. Use --force to override.`, ); } const ids = new Set(entries.map((entry) => entry.streamId)); const recentLabels = (await queryArchivedReviewLabels(mongoConnection, { limit: 1_000, nowMs: options?.nowMs, })).filter((label) => ids.has(label.evalStreamId)); const worstCases = [...entries] .filter((entry) => getLlmOverall(entry) !== null) .sort( (left, right) => (getLlmOverall(left) ?? Number.POSITIVE_INFINITY) - (getLlmOverall(right) ?? Number.POSITIVE_INFINITY), ) .slice(0, 5); return { slice, recentLabels, rootCauses: summarizeRootCauses(entries), worstCases: worstCases.length > 0 ? worstCases : entries.slice(0, 5), currentPromptSnapshot: buildCurrentPromptSnapshot(slice, entries[0]), }; } export async function writePromptVariantProposal( proposal: PromptVariantProposal, outputDir = "src/eval/proposals", ): Promise { const outputPath = resolve(process.cwd(), outputDir, `${proposal.variantId}.json`); await mkdir(dirname(outputPath), { recursive: true }); const manifest: PromptVariantProposal = { ...proposal, trustTier: proposal.trustTier ?? "T1", }; const g1 = runG1Static(manifest); const g4 = runG4Permission( manifest, createBuiltInToolRegistry(), SkillRegistry.load(resolve(process.cwd(), "src", "skills")), ); const gatedManifest: PromptVariantProposal = { ...manifest, trustTier: g1.passed && g4.passed ? promoteTrustTier(manifest.trustTier, "T2") : manifest.trustTier, }; await writeFile(`${outputPath}`, `${JSON.stringify(gatedManifest, null, 2)}\n`, "utf8"); return outputPath; }