Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
6f09408423
57 files changed
+7060
-142
No files matched your search
@@ -1,3 +1,6 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession, PromptContext } from "../types.js";
|
||||
@@ -10,11 +13,104 @@ import {
|
||||
} from "./layers.js";
|
||||
import { buildToolPolicy } from "./tool-policy.js";
|
||||
|
||||
export type PromptLayerName =
|
||||
| "context"
|
||||
| "examples"
|
||||
| "persona"
|
||||
| "policy"
|
||||
| "state"
|
||||
| "tool-policy";
|
||||
|
||||
export type PromptLayerOperation = "append" | "remove" | "replace";
|
||||
|
||||
export interface PromptLayerDiff {
|
||||
layer: PromptLayerName;
|
||||
op: PromptLayerOperation;
|
||||
reason?: string | undefined;
|
||||
text?: string | undefined;
|
||||
}
|
||||
|
||||
export interface PromptVariantManifest {
|
||||
description?: string | undefined;
|
||||
diff: PromptLayerDiff[];
|
||||
expectedImprovement?: string | undefined;
|
||||
rationale?: string | undefined;
|
||||
variantId: string;
|
||||
}
|
||||
|
||||
const variantCache = new Map<string, PromptVariantManifest | null>();
|
||||
|
||||
function getLayerTarget(layer: PromptLayerName): Exclude<PromptLayerName, "tool-policy"> {
|
||||
return layer === "tool-policy" ? "policy" : layer;
|
||||
}
|
||||
|
||||
function applyLayerDiff(
|
||||
layers: Record<Exclude<PromptLayerName, "tool-policy">, string>,
|
||||
diff: PromptLayerDiff,
|
||||
): void {
|
||||
const target = getLayerTarget(diff.layer);
|
||||
|
||||
if (diff.op === "remove") {
|
||||
layers[target] = "";
|
||||
return;
|
||||
}
|
||||
|
||||
const text = diff.text?.trim();
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (diff.op === "replace") {
|
||||
layers[target] = text;
|
||||
return;
|
||||
}
|
||||
|
||||
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
|
||||
}
|
||||
|
||||
export function loadPromptVariantManifest(
|
||||
variantId: string,
|
||||
): PromptVariantManifest | null {
|
||||
const normalized = variantId.trim();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (variantCache.has(normalized)) {
|
||||
return variantCache.get(normalized) ?? null;
|
||||
}
|
||||
|
||||
const path = resolve(
|
||||
process.cwd(),
|
||||
"src",
|
||||
"eval",
|
||||
"proposals",
|
||||
`${normalized}.json`,
|
||||
);
|
||||
if (!existsSync(path)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
|
||||
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
variantCache.set(normalized, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
variantCache.set(normalized, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class PromptBuilder {
|
||||
private constructor(
|
||||
private readonly derived: DerivedState,
|
||||
private readonly session: OnboardingSession | null,
|
||||
private readonly context: PromptContext,
|
||||
private readonly variant?: PromptVariantManifest | undefined,
|
||||
) {}
|
||||
|
||||
static forDerived(
|
||||
@@ -25,7 +121,18 @@ export class PromptBuilder {
|
||||
return new PromptBuilder(derived, session, context);
|
||||
}
|
||||
|
||||
build(): ChatMessage[] {
|
||||
applyVariant(variant: PromptVariantManifest | string): PromptBuilder {
|
||||
const manifest =
|
||||
typeof variant === "string" ? loadPromptVariantManifest(variant) : variant;
|
||||
return new PromptBuilder(
|
||||
this.derived,
|
||||
this.session,
|
||||
this.context,
|
||||
manifest ?? undefined,
|
||||
);
|
||||
}
|
||||
|
||||
buildSystemPrompt(): string {
|
||||
const promptHistory = this.context.summary
|
||||
? this.context.conversationHistory.slice(-5)
|
||||
: this.context.conversationHistory;
|
||||
@@ -34,21 +141,41 @@ export class PromptBuilder {
|
||||
conversationHistory: promptHistory,
|
||||
};
|
||||
|
||||
const systemPrompt = [
|
||||
personaLayer(),
|
||||
policyLayer(this.derived.intent, {
|
||||
const layers = {
|
||||
persona: personaLayer(),
|
||||
policy: policyLayer(this.derived.intent, {
|
||||
toolPolicy:
|
||||
this.derived.responseMode === "llm-with-tools"
|
||||
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
|
||||
: undefined,
|
||||
}),
|
||||
stateLayer(this.session, this.derived),
|
||||
examplesLayer(this.derived.intent),
|
||||
contextLayer(promptContext),
|
||||
].join("\n\n");
|
||||
state: stateLayer(this.session, this.derived),
|
||||
examples: examplesLayer(this.derived.intent),
|
||||
context: contextLayer(promptContext),
|
||||
};
|
||||
|
||||
for (const diff of this.variant?.diff ?? []) {
|
||||
applyLayerDiff(layers, diff);
|
||||
}
|
||||
|
||||
return [
|
||||
{ role: "system", content: systemPrompt },
|
||||
layers.persona,
|
||||
layers.policy,
|
||||
layers.state,
|
||||
layers.examples,
|
||||
layers.context,
|
||||
]
|
||||
.filter((layer) => layer.trim().length > 0)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
build(): ChatMessage[] {
|
||||
const promptHistory = this.context.summary
|
||||
? this.context.conversationHistory.slice(-5)
|
||||
: this.context.conversationHistory;
|
||||
|
||||
return [
|
||||
{ role: "system", content: this.buildSystemPrompt() },
|
||||
...promptHistory.map((message) => ({
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
|
||||
Reference in new issue
Block a user