Files
lolympus-agent/src/eval/loop/propose.ts
T

384 lines
11 KiB
TypeScript

import { createHash } from "node:crypto";
import { mkdir, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { z } from "zod";
import type { LlmClient } from "@llm/llm-client";
import { SkillRegistry } from "@skills/skill-loader";
import { createBuiltInToolRegistry } from "@tools/builtins/index";
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
import {
PromptBuilder,
type PromptLayerDiff,
type PromptLayerName,
type PromptVariantManifest,
} from "@skills/onboarding/prompts/builder";
import {
getEvalTournamentMinCases,
getLlmProposeMaxTokens,
getLlmProposeModel,
} from "../../config.js";
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
import { getEvalArchiveCollection } from "../archive.js";
import type { SliceKey } from "../aggregator.js";
import { queryArchivedReviewLabels } from "../review/label-archive.js";
import type { ReviewLabel } from "../review/queue.js";
import { runG1Static, runG4Permission } from "../governance/gates.js";
import { promoteTrustTier } from "../governance/trust.js";
export interface ProposalInput {
slice: SliceKey;
recentLabels: ReviewLabel[];
rootCauses?: RootCauseReport[] | undefined;
worstCases: EvalArchiveEntry[];
currentPromptSnapshot: string;
}
export interface RootCauseReport {
avgScoreLoss: number;
count: number;
examples: string[];
rootCauseStep: string;
}
export interface PromptVariantProposal extends PromptVariantManifest {
description: string;
expectedImprovement: string;
rationale: string;
}
export interface ToolProposal {
description: string;
schema: unknown;
toolName: string;
variantId: string;
whyNeeded: string;
}
const promptDiffSchema = z.object({
layer: z.enum([
"context",
"examples",
"persona",
"policy",
"state",
"tool-policy",
]),
op: z.enum(["append", "remove", "replace"]),
reason: z.string().trim().max(500).optional(),
text: z.string().trim().max(4_000).optional(),
});
const proposalSchema = z.object({
prompts: z
.array(
z.object({
description: z.string().trim().min(1).max(300),
diff: z.array(promptDiffSchema).min(1).max(10),
expectedImprovement: z.string().trim().min(1).max(500),
rationale: z.string().trim().min(1).max(500),
}),
)
.max(5),
tools: z
.array(
z.object({
description: z.string().trim().min(1).max(300),
schema: z.unknown(),
toolName: z.string().trim().min(1).max(80),
whyNeeded: z.string().trim().min(1).max(500),
}),
)
.max(5)
.default([]),
});
function stableHash(value: unknown): string {
return createHash("sha256")
.update(JSON.stringify(value))
.digest("hex")
.slice(0, 8);
}
function normalizePromptDiff(diff: z.infer<typeof promptDiffSchema>): PromptLayerDiff {
const normalized: PromptLayerDiff = {
layer: diff.layer as PromptLayerName,
op: diff.op,
};
if (diff.reason !== undefined) {
normalized.reason = diff.reason;
}
if (diff.text !== undefined) {
normalized.text = diff.text;
}
return normalized;
}
function getLlmOverall(entry: EvalArchiveEntry): number | null {
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
}
function emptyBlackboard(): Blackboard {
return {
flaggedIssues: [],
openQuestions: [],
recentNicknameAttempts: [],
toneMemory: {},
};
}
function derivedFromSlice(slice: SliceKey): DerivedState {
return {
allowShortResponse: false,
intent: { kind: slice.intent } as Intent,
phase: { kind: slice.phase } as OnboardingPhase,
responseMode: "llm",
shouldRespond: true,
suggestedMaxTokens: 384,
tier: slice.tier,
};
}
function defaultPromptContext(): PromptContext {
return {
blackboard: emptyBlackboard(),
conversationHistory: [],
userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
};
}
export function buildCurrentPromptSnapshot(
slice: SliceKey,
archiveEntry?: EvalArchiveEntry,
): string {
return PromptBuilder.forDerived(
archiveEntry?.derived ?? derivedFromSlice(slice),
null,
defaultPromptContext(),
).buildSystemPrompt();
}
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
return {
streamId: entry.streamId,
request: entry.request,
response: entry.response.text,
validation: entry.validation,
llmJudge: entry.llmJudge,
processJudge: entry.processJudge,
process: entry.process,
evaluatedAt: entry.evaluatedAt.toISOString(),
};
}
function round(value: number): number {
return Number(value.toFixed(3));
}
export function summarizeRootCauses(
entries: EvalArchiveEntry[],
): RootCauseReport[] {
const groups = new Map<
string,
{ examples: Array<{ loss: number; streamId: string }>; lossSum: number }
>();
for (const entry of entries) {
const rootCauseStep = entry.processJudge?.rootCauseStep;
const rootCauseScoreLoss = entry.processJudge?.rootCauseScoreLoss ?? 0;
if (!rootCauseStep || rootCauseScoreLoss <= 0) {
continue;
}
const group = groups.get(rootCauseStep) ?? {
examples: [],
lossSum: 0,
};
group.examples.push({
loss: rootCauseScoreLoss,
streamId: entry.streamId,
});
group.lossSum += rootCauseScoreLoss;
groups.set(rootCauseStep, group);
}
return [...groups.entries()]
.map(([rootCauseStep, group]) => ({
avgScoreLoss: round(group.lossSum / group.examples.length),
count: group.examples.length,
examples: group.examples
.sort((left, right) => right.loss - left.loss)
.slice(0, 3)
.map((example) => example.streamId),
rootCauseStep,
}))
.sort(
(left, right) =>
right.count - left.count ||
right.avgScoreLoss - left.avgScoreLoss ||
left.rootCauseStep.localeCompare(right.rootCauseStep),
)
.slice(0, 5);
}
function buildProposalPrompt(input: ProposalInput): string {
return [
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
"반드시 JSON object만 출력하세요.",
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
"",
"[slice]",
JSON.stringify(input.slice),
"",
"[recent_human_labels]",
JSON.stringify(input.recentLabels.slice(0, 20)),
"",
"[root_causes]",
JSON.stringify(input.rootCauses ?? []),
"",
"[worst_cases]",
JSON.stringify(input.worstCases.slice(0, 5).map(summarizeArchiveEntry)),
"",
"[current_prompt_snapshot]",
input.currentPromptSnapshot,
"",
'JSON schema: {"prompts":[{"description":"...","diff":[{"layer":"policy","op":"append","text":"...","reason":"..."}],"rationale":"...","expectedImprovement":"..."}],"tools":[{"description":"...","toolName":"...","schema":{},"whyNeeded":"..."}]}',
].join("\n");
}
export async function proposeVariants(
llmClient: LlmClient,
input: ProposalInput,
): Promise<{ prompts: PromptVariantProposal[]; tools: ToolProposal[] }> {
if (!llmClient.hasEndpoints()) {
throw new Error("Cannot propose variants without an LLM endpoint");
}
const response = await llmClient.chatCompletion(
[
{
role: "system",
content:
"당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.",
},
{
role: "user",
content: buildProposalPrompt(input),
},
],
{
jsonMode: true,
maxTokens: getLlmProposeMaxTokens(),
modelOverride: getLlmProposeModel(),
temperature: 0.2,
},
);
if (!response) {
throw new Error("Proposal LLM call failed");
}
const parsed = proposalSchema.parse(JSON.parse(response.content));
const prompts = parsed.prompts.map((proposal) => {
const diff = proposal.diff.map(normalizePromptDiff);
return {
...proposal,
diff,
variantId: stableHash({
description: proposal.description,
diff,
rationale: proposal.rationale,
}),
};
});
const tools = parsed.tools.map((proposal) => ({
...proposal,
variantId: stableHash(proposal),
}));
return {
prompts,
tools,
};
}
export async function buildProposalInputFromArchive(
mongoConnection: MongoConnectionLike,
slice: SliceKey,
options?: {
force?: boolean | undefined;
limit?: number | undefined;
minCases?: number | undefined;
nowMs?: number | undefined;
},
): Promise<ProposalInput> {
const limit = options?.limit ?? 50;
const entries = await getEvalArchiveCollection(mongoConnection)
.find({
tier: slice.tier,
"derived.intent.kind": slice.intent,
"derived.phase.kind": slice.phase,
})
.sort({ evaluatedAt: -1 })
.limit(limit)
.toArray();
const minCases = options?.minCases ?? getEvalTournamentMinCases();
if (!options?.force && entries.length < minCases) {
throw new Error(
`Not enough archived cases for propose: ${entries.length}/${minCases}. Use --force to override.`,
);
}
const ids = new Set(entries.map((entry) => entry.streamId));
const recentLabels = (await queryArchivedReviewLabels(mongoConnection, {
limit: 1_000,
nowMs: options?.nowMs,
})).filter((label) => ids.has(label.evalStreamId));
const worstCases = [...entries]
.filter((entry) => getLlmOverall(entry) !== null)
.sort(
(left, right) =>
(getLlmOverall(left) ?? Number.POSITIVE_INFINITY) -
(getLlmOverall(right) ?? Number.POSITIVE_INFINITY),
)
.slice(0, 5);
return {
slice,
recentLabels,
rootCauses: summarizeRootCauses(entries),
worstCases: worstCases.length > 0 ? worstCases : entries.slice(0, 5),
currentPromptSnapshot: buildCurrentPromptSnapshot(slice, entries[0]),
};
}
export async function writePromptVariantProposal(
proposal: PromptVariantProposal,
outputDir = "src/eval/proposals",
): Promise<string> {
const outputPath = resolve(process.cwd(), outputDir, `${proposal.variantId}.json`);
await mkdir(dirname(outputPath), { recursive: true });
const manifest: PromptVariantProposal = {
...proposal,
trustTier: proposal.trustTier ?? "T1",
};
const g1 = runG1Static(manifest);
const g4 = runG4Permission(
manifest,
createBuiltInToolRegistry(),
SkillRegistry.load(resolve(process.cwd(), "src", "skills")),
);
const gatedManifest: PromptVariantProposal = {
...manifest,
trustTier:
g1.passed && g4.passed
? promoteTrustTier(manifest.trustTier, "T2")
: manifest.trustTier,
};
await writeFile(`${outputPath}`, `${JSON.stringify(gatedManifest, null, 2)}\n`, "utf8");
return outputPath;
}