Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -0,0 +1,300 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { DerivedState, Intent, OnboardingPhase } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { Blackboard, PromptContext } from "@skills/onboarding/types";
|
||||
import {
|
||||
PromptBuilder,
|
||||
type PromptLayerDiff,
|
||||
type PromptLayerName,
|
||||
type PromptVariantManifest,
|
||||
} from "@skills/onboarding/prompts/builder";
|
||||
import {
|
||||
getEvalTournamentMinCases,
|
||||
getLlmProposeMaxTokens,
|
||||
getLlmProposeModel,
|
||||
} from "../../config.js";
|
||||
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
||||
import { getEvalArchiveCollection } from "../archive.js";
|
||||
import type { SliceKey } from "../aggregator.js";
|
||||
import { queryArchivedReviewLabels } from "../review/label-archive.js";
|
||||
import type { ReviewLabel } from "../review/queue.js";
|
||||
|
||||
export interface ProposalInput {
|
||||
slice: SliceKey;
|
||||
recentLabels: ReviewLabel[];
|
||||
worstCases: EvalArchiveEntry[];
|
||||
currentPromptSnapshot: string;
|
||||
}
|
||||
|
||||
export interface PromptVariantProposal extends PromptVariantManifest {
|
||||
description: string;
|
||||
expectedImprovement: string;
|
||||
rationale: string;
|
||||
}
|
||||
|
||||
export interface ToolProposal {
|
||||
description: string;
|
||||
schema: unknown;
|
||||
toolName: string;
|
||||
variantId: string;
|
||||
whyNeeded: string;
|
||||
}
|
||||
|
||||
const promptDiffSchema = z.object({
|
||||
layer: z.enum([
|
||||
"context",
|
||||
"examples",
|
||||
"persona",
|
||||
"policy",
|
||||
"state",
|
||||
"tool-policy",
|
||||
]),
|
||||
op: z.enum(["append", "remove", "replace"]),
|
||||
reason: z.string().trim().max(500).optional(),
|
||||
text: z.string().trim().max(4_000).optional(),
|
||||
});
|
||||
|
||||
const proposalSchema = z.object({
|
||||
prompts: z
|
||||
.array(
|
||||
z.object({
|
||||
description: z.string().trim().min(1).max(300),
|
||||
diff: z.array(promptDiffSchema).min(1).max(10),
|
||||
expectedImprovement: z.string().trim().min(1).max(500),
|
||||
rationale: z.string().trim().min(1).max(500),
|
||||
}),
|
||||
)
|
||||
.max(5),
|
||||
tools: z
|
||||
.array(
|
||||
z.object({
|
||||
description: z.string().trim().min(1).max(300),
|
||||
schema: z.unknown(),
|
||||
toolName: z.string().trim().min(1).max(80),
|
||||
whyNeeded: z.string().trim().min(1).max(500),
|
||||
}),
|
||||
)
|
||||
.max(5)
|
||||
.default([]),
|
||||
});
|
||||
|
||||
function stableHash(value: unknown): string {
|
||||
return createHash("sha256")
|
||||
.update(JSON.stringify(value))
|
||||
.digest("hex")
|
||||
.slice(0, 8);
|
||||
}
|
||||
|
||||
function normalizePromptDiff(diff: z.infer<typeof promptDiffSchema>): PromptLayerDiff {
|
||||
const normalized: PromptLayerDiff = {
|
||||
layer: diff.layer as PromptLayerName,
|
||||
op: diff.op,
|
||||
};
|
||||
if (diff.reason !== undefined) {
|
||||
normalized.reason = diff.reason;
|
||||
}
|
||||
if (diff.text !== undefined) {
|
||||
normalized.text = diff.text;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function getLlmOverall(entry: EvalArchiveEntry): number | null {
|
||||
return entry.llmJudge.skipped ? null : entry.llmJudge.scores.overall;
|
||||
}
|
||||
|
||||
function emptyBlackboard(): Blackboard {
|
||||
return {
|
||||
flaggedIssues: [],
|
||||
openQuestions: [],
|
||||
recentNicknameAttempts: [],
|
||||
toneMemory: {},
|
||||
};
|
||||
}
|
||||
|
||||
function derivedFromSlice(slice: SliceKey): DerivedState {
|
||||
return {
|
||||
allowShortResponse: false,
|
||||
intent: { kind: slice.intent } as Intent,
|
||||
phase: { kind: slice.phase } as OnboardingPhase,
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 384,
|
||||
tier: slice.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function defaultPromptContext(): PromptContext {
|
||||
return {
|
||||
blackboard: emptyBlackboard(),
|
||||
conversationHistory: [],
|
||||
userMessage: "현재 slice의 약점을 개선할 수 있는 기본 응답 정책을 확인해줘.",
|
||||
};
|
||||
}
|
||||
|
||||
export function buildCurrentPromptSnapshot(
|
||||
slice: SliceKey,
|
||||
archiveEntry?: EvalArchiveEntry,
|
||||
): string {
|
||||
return PromptBuilder.forDerived(
|
||||
archiveEntry?.derived ?? derivedFromSlice(slice),
|
||||
null,
|
||||
defaultPromptContext(),
|
||||
).buildSystemPrompt();
|
||||
}
|
||||
|
||||
function summarizeArchiveEntry(entry: EvalArchiveEntry): Record<string, unknown> {
|
||||
return {
|
||||
streamId: entry.streamId,
|
||||
request: entry.request,
|
||||
response: entry.response.text,
|
||||
validation: entry.validation,
|
||||
llmJudge: entry.llmJudge,
|
||||
processJudge: entry.processJudge,
|
||||
process: entry.process,
|
||||
evaluatedAt: entry.evaluatedAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function buildProposalPrompt(input: ProposalInput): string {
|
||||
return [
|
||||
"온보딩 에이전트의 약점 slice를 개선할 prompt/tool variant를 제안해 주세요.",
|
||||
"반드시 JSON object만 출력하세요.",
|
||||
"prompt diff는 기존 system prompt layer에 append/replace/remove 중 하나로 적용됩니다.",
|
||||
"과도한 일반론보다 아래 실패 케이스의 반복 원인을 직접 줄이는 작은 변경을 선호하세요.",
|
||||
"",
|
||||
"[slice]",
|
||||
JSON.stringify(input.slice),
|
||||
"",
|
||||
"[recent_human_labels]",
|
||||
JSON.stringify(input.recentLabels.slice(0, 20)),
|
||||
"",
|
||||
"[worst_cases]",
|
||||
JSON.stringify(input.worstCases.slice(0, 5).map(summarizeArchiveEntry)),
|
||||
"",
|
||||
"[current_prompt_snapshot]",
|
||||
input.currentPromptSnapshot,
|
||||
"",
|
||||
'JSON schema: {"prompts":[{"description":"...","diff":[{"layer":"policy","op":"append","text":"...","reason":"..."}],"rationale":"...","expectedImprovement":"..."}],"tools":[{"description":"...","toolName":"...","schema":{},"whyNeeded":"..."}]}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function proposeVariants(
|
||||
llmClient: LlmClient,
|
||||
input: ProposalInput,
|
||||
): Promise<{ prompts: PromptVariantProposal[]; tools: ToolProposal[] }> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
throw new Error("Cannot propose variants without an LLM endpoint");
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 에이전트 품질 개선을 돕는 prompt/tool reviewer입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildProposalPrompt(input),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: getLlmProposeMaxTokens(),
|
||||
modelOverride: getLlmProposeModel(),
|
||||
temperature: 0.2,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
throw new Error("Proposal LLM call failed");
|
||||
}
|
||||
|
||||
const parsed = proposalSchema.parse(JSON.parse(response.content));
|
||||
const prompts = parsed.prompts.map((proposal) => {
|
||||
const diff = proposal.diff.map(normalizePromptDiff);
|
||||
return {
|
||||
...proposal,
|
||||
diff,
|
||||
variantId: stableHash({
|
||||
description: proposal.description,
|
||||
diff,
|
||||
rationale: proposal.rationale,
|
||||
}),
|
||||
};
|
||||
});
|
||||
const tools = parsed.tools.map((proposal) => ({
|
||||
...proposal,
|
||||
variantId: stableHash(proposal),
|
||||
}));
|
||||
|
||||
return {
|
||||
prompts,
|
||||
tools,
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildProposalInputFromArchive(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
slice: SliceKey,
|
||||
options?: {
|
||||
force?: boolean | undefined;
|
||||
limit?: number | undefined;
|
||||
minCases?: number | undefined;
|
||||
nowMs?: number | undefined;
|
||||
},
|
||||
): Promise<ProposalInput> {
|
||||
const limit = options?.limit ?? 50;
|
||||
const entries = await getEvalArchiveCollection(mongoConnection)
|
||||
.find({
|
||||
tier: slice.tier,
|
||||
"derived.intent.kind": slice.intent,
|
||||
"derived.phase.kind": slice.phase,
|
||||
})
|
||||
.sort({ evaluatedAt: -1 })
|
||||
.limit(limit)
|
||||
.toArray();
|
||||
const minCases = options?.minCases ?? getEvalTournamentMinCases();
|
||||
|
||||
if (!options?.force && entries.length < minCases) {
|
||||
throw new Error(
|
||||
`Not enough archived cases for propose: ${entries.length}/${minCases}. Use --force to override.`,
|
||||
);
|
||||
}
|
||||
|
||||
const ids = new Set(entries.map((entry) => entry.streamId));
|
||||
const recentLabels = (await queryArchivedReviewLabels(mongoConnection, {
|
||||
limit: 1_000,
|
||||
nowMs: options?.nowMs,
|
||||
})).filter((label) => ids.has(label.evalStreamId));
|
||||
const worstCases = [...entries]
|
||||
.filter((entry) => getLlmOverall(entry) !== null)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
(getLlmOverall(left) ?? Number.POSITIVE_INFINITY) -
|
||||
(getLlmOverall(right) ?? Number.POSITIVE_INFINITY),
|
||||
)
|
||||
.slice(0, 5);
|
||||
|
||||
return {
|
||||
slice,
|
||||
recentLabels,
|
||||
worstCases: worstCases.length > 0 ? worstCases : entries.slice(0, 5),
|
||||
currentPromptSnapshot: buildCurrentPromptSnapshot(slice, entries[0]),
|
||||
};
|
||||
}
|
||||
|
||||
export async function writePromptVariantProposal(
|
||||
proposal: PromptVariantProposal,
|
||||
outputDir = "src/eval/proposals",
|
||||
): Promise<string> {
|
||||
const outputPath = resolve(process.cwd(), outputDir, `${proposal.variantId}.json`);
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(`${outputPath}`, `${JSON.stringify(proposal, null, 2)}\n`, "utf8");
|
||||
return outputPath;
|
||||
}
|
||||
Reference in new issue
Block a user