Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit 6f09408423
57 files changed
+7060 -142

No files matched your search

+136 -9
View File
@@ -1,3 +1,6 @@
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import type { ChatMessage } from "@llm/types";
import type { DerivedState } from "../state/onboarding-state.js";
import type { OnboardingSession, PromptContext } from "../types.js";
@@ -10,11 +13,104 @@ import {
} from "./layers.js";
import { buildToolPolicy } from "./tool-policy.js";
export type PromptLayerName =
| "context"
| "examples"
| "persona"
| "policy"
| "state"
| "tool-policy";
export type PromptLayerOperation = "append" | "remove" | "replace";
export interface PromptLayerDiff {
layer: PromptLayerName;
op: PromptLayerOperation;
reason?: string | undefined;
text?: string | undefined;
}
export interface PromptVariantManifest {
description?: string | undefined;
diff: PromptLayerDiff[];
expectedImprovement?: string | undefined;
rationale?: string | undefined;
variantId: string;
}
const variantCache = new Map<string, PromptVariantManifest | null>();
function getLayerTarget(layer: PromptLayerName): Exclude<PromptLayerName, "tool-policy"> {
return layer === "tool-policy" ? "policy" : layer;
}
function applyLayerDiff(
layers: Record<Exclude<PromptLayerName, "tool-policy">, string>,
diff: PromptLayerDiff,
): void {
const target = getLayerTarget(diff.layer);
if (diff.op === "remove") {
layers[target] = "";
return;
}
const text = diff.text?.trim();
if (!text) {
return;
}
if (diff.op === "replace") {
layers[target] = text;
return;
}
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
}
export function loadPromptVariantManifest(
variantId: string,
): PromptVariantManifest | null {
const normalized = variantId.trim();
if (!normalized) {
return null;
}
if (variantCache.has(normalized)) {
return variantCache.get(normalized) ?? null;
}
const path = resolve(
process.cwd(),
"src",
"eval",
"proposals",
`${normalized}.json`,
);
if (!existsSync(path)) {
variantCache.set(normalized, null);
return null;
}
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
variantCache.set(normalized, null);
return null;
}
variantCache.set(normalized, parsed);
return parsed;
} catch {
variantCache.set(normalized, null);
return null;
}
}
export class PromptBuilder {
private constructor(
private readonly derived: DerivedState,
private readonly session: OnboardingSession | null,
private readonly context: PromptContext,
private readonly variant?: PromptVariantManifest | undefined,
) {}
static forDerived(
@@ -25,7 +121,18 @@ export class PromptBuilder {
return new PromptBuilder(derived, session, context);
}
build(): ChatMessage[] {
applyVariant(variant: PromptVariantManifest | string): PromptBuilder {
const manifest =
typeof variant === "string" ? loadPromptVariantManifest(variant) : variant;
return new PromptBuilder(
this.derived,
this.session,
this.context,
manifest ?? undefined,
);
}
buildSystemPrompt(): string {
const promptHistory = this.context.summary
? this.context.conversationHistory.slice(-5)
: this.context.conversationHistory;
@@ -34,21 +141,41 @@ export class PromptBuilder {
conversationHistory: promptHistory,
};
const systemPrompt = [
personaLayer(),
policyLayer(this.derived.intent, {
const layers = {
persona: personaLayer(),
policy: policyLayer(this.derived.intent, {
toolPolicy:
this.derived.responseMode === "llm-with-tools"
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
: undefined,
}),
stateLayer(this.session, this.derived),
examplesLayer(this.derived.intent),
contextLayer(promptContext),
].join("\n\n");
state: stateLayer(this.session, this.derived),
examples: examplesLayer(this.derived.intent),
context: contextLayer(promptContext),
};
for (const diff of this.variant?.diff ?? []) {
applyLayerDiff(layers, diff);
}
return [
{ role: "system", content: systemPrompt },
layers.persona,
layers.policy,
layers.state,
layers.examples,
layers.context,
]
.filter((layer) => layer.trim().length > 0)
.join("\n\n");
}
build(): ChatMessage[] {
const promptHistory = this.context.summary
? this.context.conversationHistory.slice(-5)
: this.context.conversationHistory;
return [
{ role: "system", content: this.buildSystemPrompt() },
...promptHistory.map((message) => ({
role: message.role,
content: message.content,