384 lines
11 KiB
TypeScript
384 lines
11 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import { dirname, resolve } from "node:path";
|
|
|
|
import { z } from "zod";
|
|
import type { LlmClient } from "@llm/llm-client";
|
|
import { SkillRegistry } from "@skills/skill-loader";
|
|
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
|
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
|
|
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
|
import {
|
|
PromptBuilder,
|
|
type PromptLayerDiff,
|
|
type PromptLayerName,
|
|
type PromptVariantManifest,
|
|
} from "@skills/onboarding/prompts/builder";
|
|
import {
|
|
getEvalTournamentMinCases,
|
|
getLlmProposeMaxTokens,
|
|
getLlmProposeModel,
|
|
} from "../../config.js";
|
|
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
|
import { getEvalArchiveCollection } from "../archive.js";
|
|
import type { SliceKey } from "../aggregator.js";
|
|
import { queryArchivedReviewLabels } from "../review/label-archive.js";
|
|
import type { ReviewLabel } from "../review/queue.js";
|
|
import { runG1Static, runG4Permission } from "../governance/gates.js";
|
|
import { promoteTrustTier } from "../governance/trust.js";
|
|
|
|
export interface ProposalInput {
|
|
slice: SliceKey;
|
|
recentLabels: ReviewLabel[];
|
|
rootCauses?: RootCauseReport[] | undefined;
|
|
worstCases: EvalArchiveEntry[];
|
|
currentPromptSnapshot: string;
|
|
}
|
|
|
|
export interface RootCauseReport {
|
|
avgScoreLoss: number;
|
|
count: number;
|
|
examples: string[];
|
|
rootCauseStep: string;
|
|
}
|
|
|
|
export interface PromptVariantProposal extends PromptVariantManifest {
|
|
description: string;
|
|
expectedImprovement: string;
|
|
rationale: string;
|
|
}
|
|
|
|
export interface ToolProposal {
|
|
description: string;
|
|
schema: unknown;
|
|
toolName: string;
|
|
variantId: string;
|
|
whyNeeded: string;
|
|
}
|
|
|
|
const promptDiffSchema = z.object({
|
|
layer: z.enum([
|
|
"context",
|
|
"examples",
|
|
"persona",
|
|
"policy",
|
|
"state",
|
|
"tool-policy",
|
|
]),
|
|
op: z.enum(["append", "remove", "replace"]),
|
|
reason: z.string().trim().max(500).optional(),
|
|
text: z.string().trim().max(4_000).optional(),
|
|
});
|
|
|
|
const proposalSchema = z.object({
|
|
prompts: z
|
|
.array(
|
|
z.object({
|
|
description: z.string().trim().min(1).max(300),
|
|
diff: z.array(promptDiffSchema).min(1).max(10),
|
|
expectedImprovement: z.string().trim().min(1).max(500),
|
|
rationale: z.string().trim().min(1).max(500),
|
|
}),
|
|
)
|
|
.max(5),
|
|
tools: z
|
|
.array(
|
|
z.object({
|
|
description: z.string().trim().min(1).max(300),
|
|
schema: z.unknown(),
|
|
toolName: z.string().trim().min(1).max(80),
|
|
whyNeeded: z.string().trim().min(1).max(500),
|
|
}),
|
|
)
|
|
.max(5)
|
|
.default([]),
|
|
});
|
|
|
|
function stableHash(value: unknown): string {
|
|
return createHash("sha256")
|
|
.update(JSON.stringify(value))
|
|
.digest("hex")
|
|
.slice(0, 8);
|
|
}
|
|
|
|
function normalizePromptDiff(diff: z.infer<typeof promptDiffSchema>): PromptLayerDiff {
|
|
const normalized: PromptLayerDiff = {
|
|
layer: diff.layer as PromptLayerName,
|
|
op: diff.op,
|
|
};
|
|
if (diff.reason !== undefined) {
|
|
normalized.reason = diff.reason;
|
|
}
|
|
if (diff.text !== undefined) {
|
|
normalized.text = diff.text;
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
|
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
|
}
|
|
|
|
function emptyBlackboard(): Blackboard {
|
|
return {
|
|
flaggedIssues: [],
|
|
openQuestions: [],
|
|
recentNicknameAttempts: [],
|
|
toneMemory: {},
|
|
};
|
|
}
|
|
|
|
function derivedFromSlice(slice: SliceKey): DerivedState {
|
|
return {
|
|
allowShortResponse: false,
|
|
intent: { kind: slice.intent } as Intent,
|
|
phase: { kind: slice.phase } as OnboardingPhase,
|
|
responseMode: "llm",
|
|
shouldRespond: true,
|
|
suggestedMaxTokens: 384,
|
|
tier: slice.tier,
|
|
};
|
|
}
|
|
|
|
function defaultPromptContext(): PromptContext {
|
|
return {
|
|
blackboard: emptyBlackboard(),
|
|
conversationHistory: [],
|
|
userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
|
};
|
|
}
|
|
|
|
export function buildCurrentPromptSnapshot(
|
|
slice: SliceKey,
|
|
archiveEntry?: EvalArchiveEntry,
|
|
): string {
|
|
return PromptBuilder.forDerived(
|
|
archiveEntry?.derived ?? derivedFromSlice(slice),
|
|
null,
|
|
defaultPromptContext(),
|
|
).buildSystemPrompt();
|
|
}
|
|
|
|
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
|
|
return {
|
|
streamId: entry.streamId,
|
|
request: entry.request,
|
|
response: entry.response.text,
|
|
validation: entry.validation,
|
|
llmJudge: entry.llmJudge,
|
|
processJudge: entry.processJudge,
|
|
process: entry.process,
|
|
evaluatedAt: entry.evaluatedAt.toISOString(),
|
|
};
|
|
}
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(3));
|
|
}
|
|
|
|
export function summarizeRootCauses(
|
|
entries: EvalArchiveEntry[],
|
|
): RootCauseReport[] {
|
|
const groups = new Map<
|
|
string,
|
|
{ examples: Array<{ loss: number; streamId: string }>; lossSum: number }
|
|
>();
|
|
|
|
for (const entry of entries) {
|
|
const rootCauseStep = entry.processJudge?.rootCauseStep;
|
|
const rootCauseScoreLoss = entry.processJudge?.rootCauseScoreLoss ?? 0;
|
|
if (!rootCauseStep || rootCauseScoreLoss <= 0) {
|
|
continue;
|
|
}
|
|
|
|
const group = groups.get(rootCauseStep) ?? {
|
|
examples: [],
|
|
lossSum: 0,
|
|
};
|
|
group.examples.push({
|
|
loss: rootCauseScoreLoss,
|
|
streamId: entry.streamId,
|
|
});
|
|
group.lossSum += rootCauseScoreLoss;
|
|
groups.set(rootCauseStep, group);
|
|
}
|
|
|
|
return [...groups.entries()]
|
|
.map(([rootCauseStep, group]) => ({
|
|
avgScoreLoss: round(group.lossSum / group.examples.length),
|
|
count: group.examples.length,
|
|
examples: group.examples
|
|
.sort((left, right) => right.loss - left.loss)
|
|
.slice(0, 3)
|
|
.map((example) => example.streamId),
|
|
rootCauseStep,
|
|
}))
|
|
.sort(
|
|
(left, right) =>
|
|
right.count - left.count ||
|
|
right.avgScoreLoss - left.avgScoreLoss ||
|
|
left.rootCauseStep.localeCompare(right.rootCauseStep),
|
|
)
|
|
.slice(0, 5);
|
|
}
|
|
|
|
function buildProposalPrompt(input: ProposalInput): string {
|
|
return [
|
|
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
|
|
"반드시 JSON object만 출력하세요.",
|
|
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
|
|
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
|
|
"",
|
|
"[slice]",
|
|
JSON.stringify(input.slice),
|
|
"",
|
|
"[recent_human_labels]",
|
|
JSON.stringify(input.recentLabels.slice(0, 20)),
|
|
"",
|
|
"[root_causes]",
|
|
JSON.stringify(input.rootCauses ?? []),
|
|
"",
|
|
"[worst_cases]",
|
|
JSON.stringify(input.worstCases.slice(0, 5).map(summarizeArchiveEntry)),
|
|
"",
|
|
"[current_prompt_snapshot]",
|
|
input.currentPromptSnapshot,
|
|
"",
|
|
'JSON schema: {"prompts":[{"description":"...","diff":[{"layer":"policy","op":"append","text":"...","reason":"..."}],"rationale":"...","expectedImprovement":"..."}],"tools":[{"description":"...","toolName":"...","schema":{},"whyNeeded":"..."}]}',
|
|
].join("\n");
|
|
}
|
|
|
|
export async function proposeVariants(
|
|
llmClient: LlmClient,
|
|
input: ProposalInput,
|
|
): Promise<{ prompts: PromptVariantProposal[]; tools: ToolProposal[] }> {
|
|
if (!llmClient.hasEndpoints()) {
|
|
throw new Error("Cannot propose variants without an LLM endpoint");
|
|
}
|
|
|
|
const response = await llmClient.chatCompletion(
|
|
[
|
|
{
|
|
role: "system",
|
|
content:
|
|
"당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.",
|
|
},
|
|
{
|
|
role: "user",
|
|
content: buildProposalPrompt(input),
|
|
},
|
|
],
|
|
{
|
|
jsonMode: true,
|
|
maxTokens: getLlmProposeMaxTokens(),
|
|
modelOverride: getLlmProposeModel(),
|
|
temperature: 0.2,
|
|
},
|
|
);
|
|
|
|
if (!response) {
|
|
throw new Error("Proposal LLM call failed");
|
|
}
|
|
|
|
const parsed = proposalSchema.parse(JSON.parse(response.content));
|
|
const prompts = parsed.prompts.map((proposal) => {
|
|
const diff = proposal.diff.map(normalizePromptDiff);
|
|
return {
|
|
...proposal,
|
|
diff,
|
|
variantId: stableHash({
|
|
description: proposal.description,
|
|
diff,
|
|
rationale: proposal.rationale,
|
|
}),
|
|
};
|
|
});
|
|
const tools = parsed.tools.map((proposal) => ({
|
|
...proposal,
|
|
variantId: stableHash(proposal),
|
|
}));
|
|
|
|
return {
|
|
prompts,
|
|
tools,
|
|
};
|
|
}
|
|
|
|
export async function buildProposalInputFromArchive(
|
|
mongoConnection: MongoConnectionLike,
|
|
slice: SliceKey,
|
|
options?: {
|
|
force?: boolean | undefined;
|
|
limit?: number | undefined;
|
|
minCases?: number | undefined;
|
|
nowMs?: number | undefined;
|
|
},
|
|
): Promise<ProposalInput> {
|
|
const limit = options?.limit ?? 50;
|
|
const entries = await getEvalArchiveCollection(mongoConnection)
|
|
.find({
|
|
tier: slice.tier,
|
|
"derived.intent.kind": slice.intent,
|
|
"derived.phase.kind": slice.phase,
|
|
})
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(limit)
|
|
.toArray();
|
|
const minCases = options?.minCases ?? getEvalTournamentMinCases();
|
|
|
|
if (!options?.force && entries.length < minCases) {
|
|
throw new Error(
|
|
`Not enough archived cases for propose: ${entries.length}/${minCases}. Use --force to override.`,
|
|
);
|
|
}
|
|
|
|
const ids = new Set(entries.map((entry) => entry.streamId));
|
|
const recentLabels = (await queryArchivedReviewLabels(mongoConnection, {
|
|
limit: 1_000,
|
|
nowMs: options?.nowMs,
|
|
})).filter((label) => ids.has(label.evalStreamId));
|
|
const worstCases = [...entries]
|
|
.filter((entry) => getLlmOverall(entry) !== null)
|
|
.sort(
|
|
(left, right) =>
|
|
(getLlmOverall(left) ?? Number.POSITIVE_INFINITY) -
|
|
(getLlmOverall(right) ?? Number.POSITIVE_INFINITY),
|
|
)
|
|
.slice(0, 5);
|
|
|
|
return {
|
|
slice,
|
|
recentLabels,
|
|
rootCauses: summarizeRootCauses(entries),
|
|
worstCases: worstCases.length > 0 ? worstCases : entries.slice(0, 5),
|
|
currentPromptSnapshot: buildCurrentPromptSnapshot(slice, entries[0]),
|
|
};
|
|
}
|
|
|
|
export async function writePromptVariantProposal(
|
|
proposal: PromptVariantProposal,
|
|
outputDir = "src/eval/proposals",
|
|
): Promise<string> {
|
|
const outputPath = resolve(process.cwd(), outputDir, `${proposal.variantId}.json`);
|
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
const manifest: PromptVariantProposal = {
|
|
...proposal,
|
|
trustTier: proposal.trustTier ?? "T1",
|
|
};
|
|
const g1 = runG1Static(manifest);
|
|
const g4 = runG4Permission(
|
|
manifest,
|
|
createBuiltInToolRegistry(),
|
|
SkillRegistry.load(resolve(process.cwd(), "src", "skills")),
|
|
);
|
|
const gatedManifest: PromptVariantProposal = {
|
|
...manifest,
|
|
trustTier:
|
|
g1.passed && g4.passed
|
|
? promoteTrustTier(manifest.trustTier, "T2")
|
|
: manifest.trustTier,
|
|
};
|
|
await writeFile(`${outputPath}`, `${JSON.stringify(gatedManifest, null, 2)}\n`, "utf8");
|
|
return outputPath;
|
|
}
|