Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit 6f09408423
57 files changed
+7060 -142

No files matched your search

+136 -9
View File
@@ -1,3 +1,6 @@
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import type { ChatMessage } from "@llm/types";
import type { DerivedState } from "../state/onboarding-state.js";
import type { OnboardingSession, PromptContext } from "../types.js";
@@ -10,11 +13,104 @@ import {
} from "./layers.js";
import { buildToolPolicy } from "./tool-policy.js";
export type PromptLayerName =
| "context"
| "examples"
| "persona"
| "policy"
| "state"
| "tool-policy";
export type PromptLayerOperation = "append" | "remove" | "replace";
export interface PromptLayerDiff {
layer: PromptLayerName;
op: PromptLayerOperation;
reason?: string | undefined;
text?: string | undefined;
}
export interface PromptVariantManifest {
description?: string | undefined;
diff: PromptLayerDiff[];
expectedImprovement?: string | undefined;
rationale?: string | undefined;
variantId: string;
}
const variantCache = new Map<string, PromptVariantManifest | null>();
function getLayerTarget(layer: PromptLayerName): Exclude<PromptLayerName, "tool-policy"> {
return layer === "tool-policy" ? "policy" : layer;
}
function applyLayerDiff(
layers: Record<Exclude<PromptLayerName, "tool-policy">, string>,
diff: PromptLayerDiff,
): void {
const target = getLayerTarget(diff.layer);
if (diff.op === "remove") {
layers[target] = "";
return;
}
const text = diff.text?.trim();
if (!text) {
return;
}
if (diff.op === "replace") {
layers[target] = text;
return;
}
layers[target] = layers[target] ? `${layers[target]}\n${text}` : text;
}
export function loadPromptVariantManifest(
variantId: string,
): PromptVariantManifest | null {
const normalized = variantId.trim();
if (!normalized) {
return null;
}
if (variantCache.has(normalized)) {
return variantCache.get(normalized) ?? null;
}
const path = resolve(
process.cwd(),
"src",
"eval",
"proposals",
`${normalized}.json`,
);
if (!existsSync(path)) {
variantCache.set(normalized, null);
return null;
}
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as PromptVariantManifest;
if (parsed.variantId !== normalized || !Array.isArray(parsed.diff)) {
variantCache.set(normalized, null);
return null;
}
variantCache.set(normalized, parsed);
return parsed;
} catch {
variantCache.set(normalized, null);
return null;
}
}
export class PromptBuilder {
private constructor(
private readonly derived: DerivedState,
private readonly session: OnboardingSession | null,
private readonly context: PromptContext,
private readonly variant?: PromptVariantManifest | undefined,
) {}
static forDerived(
@@ -25,7 +121,18 @@ export class PromptBuilder {
return new PromptBuilder(derived, session, context);
}
build(): ChatMessage[] {
applyVariant(variant: PromptVariantManifest | string): PromptBuilder {
const manifest =
typeof variant === "string" ? loadPromptVariantManifest(variant) : variant;
return new PromptBuilder(
this.derived,
this.session,
this.context,
manifest ?? undefined,
);
}
buildSystemPrompt(): string {
const promptHistory = this.context.summary
? this.context.conversationHistory.slice(-5)
: this.context.conversationHistory;
@@ -34,21 +141,41 @@ export class PromptBuilder {
conversationHistory: promptHistory,
};
const systemPrompt = [
personaLayer(),
policyLayer(this.derived.intent, {
const layers = {
persona: personaLayer(),
policy: policyLayer(this.derived.intent, {
toolPolicy:
this.derived.responseMode === "llm-with-tools"
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
: undefined,
}),
stateLayer(this.session, this.derived),
examplesLayer(this.derived.intent),
contextLayer(promptContext),
].join("\n\n");
state: stateLayer(this.session, this.derived),
examples: examplesLayer(this.derived.intent),
context: contextLayer(promptContext),
};
for (const diff of this.variant?.diff ?? []) {
applyLayerDiff(layers, diff);
}
return [
{ role: "system", content: systemPrompt },
layers.persona,
layers.policy,
layers.state,
layers.examples,
layers.context,
]
.filter((layer) => layer.trim().length > 0)
.join("\n\n");
}
build(): ChatMessage[] {
const promptHistory = this.context.summary
? this.context.conversationHistory.slice(-5)
: this.context.conversationHistory;
return [
{ role: "system", content: this.buildSystemPrompt() },
...promptHistory.map((message) => ({
role: message.role,
content: message.content,
+10
View File
@@ -6,6 +6,7 @@ import {
getTier2CriticModel,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
import { getActiveProcessTrace } from "./trace.js";
export interface CritiqueResult {
changed: boolean;
@@ -55,6 +56,7 @@ export class OnboardingCritic {
messages: ChatMessage[],
options?: { deadlineMs?: number | undefined },
): Promise<CritiqueResult> {
const traceStep = getActiveProcessTrace()?.startStep("critic");
const response = await this.llmClient.chatCompletion(
buildCriticMessages(draft, derived, messages),
{
@@ -69,12 +71,20 @@ export class OnboardingCritic {
const text = response?.content.trim() ?? "";
if (!text || text.startsWith("[OK]")) {
traceStep?.finish({
decision: "keep",
rationale: text || "empty critic response",
});
return {
changed: false,
text: draft,
};
}
traceStep?.finish({
decision: "rewrite",
rationale: text.slice(0, 200),
});
return {
changed: true,
text,
+15 -3
View File
@@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js";
import { OnboardingCritic } from "./critic.js";
import type { PlannerOutput } from "./planner-schema.js";
import { PlannerOutputSchema } from "./planner-schema.js";
import { getActiveProcessTrace } from "./trace.js";
import { runWithTools } from "./tool-loop.js";
export interface OrchestratorRunOptions {
@@ -260,6 +261,17 @@ export class OnboardingOrchestrator {
messages: ChatMessage[],
): Promise<PlannerOutput> {
const logger = getRequestLogger();
const traceStep = getActiveProcessTrace()?.startStep("plan");
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
traceStep?.finish({
approach: plan.approach,
mustAvoid: plan.mustAvoid,
mustInclude: plan.mustInclude,
suggestedTools: plan.suggestedTools,
targetLengthChars: plan.targetLengthChars,
});
return plan;
};
const plannerMessages = buildPlannerMessages(derived, messages);
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
deadlineMs: getRemainingDeadlineMs(),
@@ -283,7 +295,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner output parsed",
);
return firstPlan;
return finishTrace(firstPlan);
}
const retryMessages = [
@@ -315,7 +327,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner output parsed",
);
return retryPlan;
return finishTrace(retryPlan);
}
const fallbackPlan = getDefaultPlan(derived);
@@ -327,7 +339,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner fell back to default plan",
);
return fallbackPlan;
return finishTrace(fallbackPlan);
}
private async runTier0(
+36 -9
View File
@@ -4,6 +4,7 @@ import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
import type { Tool, ToolContext } from "@tools/tool";
import { getActiveProcessTrace } from "./trace.js";
export interface ToolLoopOptions {
deadlineMs?: number | undefined;
@@ -107,15 +108,33 @@ export async function runWithTools(
"[ToolLoop] Executing tool call",
);
const validated = tool.argsSchema.parse(call.arguments);
const result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
let result: unknown;
try {
const validated = tool.argsSchema.parse(call.arguments);
result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
traceStep?.finish({
args: call.arguments,
name: tool.name,
ok: true,
resultSummary: summarizeToolResult(result),
});
} catch (error) {
traceStep?.finish({
args: call.arguments,
name: tool.name,
ok: false,
resultSummary: error instanceof Error ? error.message : String(error),
});
throw error;
}
logger.debug(
{
@@ -137,3 +156,11 @@ export async function runWithTools(
return null;
}
function summarizeToolResult(result: unknown): string {
try {
return JSON.stringify(result).slice(0, 200);
} catch {
return String(result).slice(0, 200);
}
}
+169
View File
@@ -0,0 +1,169 @@
import { AsyncLocalStorage } from "node:async_hooks";
export type ProcessTraceStepName =
| "classify"
| "critic"
| "derive"
| "plan"
| "respond"
| "tool-call"
| "validate";
export interface ProcessTraceStep<T> {
name: ProcessTraceStepName;
startedAtMs: number;
durationMs: number;
output: T;
}
export interface ProcessTraceSnapshot {
steps: {
classify?: ProcessTraceStep<{
confidence: number;
intent: string;
source: "regex" | "llm";
}> | undefined;
derive?: ProcessTraceStep<{
phase: string;
responseMode: string;
shouldRespond: boolean;
tier: 0 | 1 | 2;
}> | undefined;
plan?: ProcessTraceStep<{
approach: string;
mustAvoid: string[];
mustInclude: string[];
suggestedTools: string[];
targetLengthChars: number;
}> | undefined;
toolCalls?: Array<
ProcessTraceStep<{
args: unknown;
name: string;
ok: boolean;
resultSummary: string;
}>
> | undefined;
respond?: ProcessTraceStep<{
draftText: string;
tokensIn: number;
tokensOut: number;
}> | undefined;
critic?: ProcessTraceStep<{
decision: "keep" | "rewrite";
rationale: string;
}> | undefined;
validate?: ProcessTraceStep<{
reasons: string[];
status: "pass" | "reject" | "soft-warn";
}> | undefined;
};
}
type ProcessTraceOutputByName = {
classify: NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"];
critic: NonNullable<ProcessTraceSnapshot["steps"]["critic"]>["output"];
derive: NonNullable<ProcessTraceSnapshot["steps"]["derive"]>["output"];
plan: NonNullable<ProcessTraceSnapshot["steps"]["plan"]>["output"];
respond: NonNullable<ProcessTraceSnapshot["steps"]["respond"]>["output"];
"tool-call": NonNullable<
NonNullable<ProcessTraceSnapshot["steps"]["toolCalls"]>[number]
>["output"];
validate: NonNullable<ProcessTraceSnapshot["steps"]["validate"]>["output"];
};
const processTraceStorage = new AsyncLocalStorage<ProcessTraceCollector>();
export class ProcessTraceCollector {
private readonly steps: ProcessTraceSnapshot["steps"] = {};
startStep<Name extends ProcessTraceStepName>(
name: Name,
): {
finish: (output: ProcessTraceOutputByName[Name]) => void;
} {
const startedAtMs = Date.now();
let finished = false;
return {
finish: (output: ProcessTraceOutputByName[Name]): void => {
if (finished) {
return;
}
finished = true;
const step: ProcessTraceStep<ProcessTraceOutputByName[Name]> = {
name,
startedAtMs,
durationMs: Math.max(0, Date.now() - startedAtMs),
output,
};
if (name === "tool-call") {
this.steps.toolCalls = [
...(this.steps.toolCalls ?? []),
step as ProcessTraceStep<
ProcessTraceOutputByName["tool-call"]
>,
];
return;
}
switch (name) {
case "classify":
this.steps.classify = step as ProcessTraceStep<
ProcessTraceOutputByName["classify"]
>;
return;
case "critic":
this.steps.critic = step as ProcessTraceStep<
ProcessTraceOutputByName["critic"]
>;
return;
case "derive":
this.steps.derive = step as ProcessTraceStep<
ProcessTraceOutputByName["derive"]
>;
return;
case "plan":
this.steps.plan = step as ProcessTraceStep<
ProcessTraceOutputByName["plan"]
>;
return;
case "respond":
this.steps.respond = step as ProcessTraceStep<
ProcessTraceOutputByName["respond"]
>;
return;
case "validate":
this.steps.validate = step as ProcessTraceStep<
ProcessTraceOutputByName["validate"]
>;
return;
}
},
};
}
snapshot(): ProcessTraceSnapshot {
return {
steps: {
...this.steps,
toolCalls: this.steps.toolCalls
? [...this.steps.toolCalls]
: undefined,
},
};
}
}
export function runWithProcessTrace<T>(
collector: ProcessTraceCollector,
fn: () => T,
): T {
return processTraceStorage.run(collector, fn);
}
export function getActiveProcessTrace(): ProcessTraceCollector | undefined {
return processTraceStorage.getStore();
}
@@ -3,6 +3,7 @@ import { getMaxResponseLength } from "../../../config.js";
import { VALID_TIER_SET } from "../constants.js";
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
import type { DerivedState } from "../state/onboarding-state.js";
import { getActiveProcessTrace } from "./trace.js";
const ALLOWED_URL_PATTERNS = [
/discord\.gg\/lolympus/iu,
@@ -118,6 +119,10 @@ function markValidation(result: ValidationResult): ValidationResult {
"agent.validator.status": result.status,
"agent.validator.warnings": result.reasons,
});
getActiveProcessTrace()?.startStep("validate").finish({
status: result.status,
reasons: result.reasons,
});
return result;
}
+1 -1
View File
@@ -61,7 +61,7 @@ export function collectNicknameAliases(
for (const source of sources) {
if (Array.isArray(source)) {
for (const value of source) {
for (const value of source as readonly string[]) {
pushAlias(value);
}
continue;
+63 -3
View File
@@ -8,6 +8,7 @@ import type { ChatMessage } from '@llm/types';
import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { createBuiltInToolRegistry } from '@tools/builtins/index';
import type { Tool, ToolContext } from '@tools/tool';
import { PromptBuilder } from './prompts/builder.js';
@@ -27,12 +28,21 @@ import {
getRemainingStatusMessage,
getTimeoutWarningMessage,
} from './onboarding-templates.js';
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from '../../config.js';
import {
getOnboardingSessionKeyPrefix,
getLlmTemperature,
getPromptVariantId,
} from '../../config.js';
import { OnboardingBlackboardStore } from './state/blackboard.js';
import { deriveState } from './state/derive.js';
import { OnboardingIntentClassifier } from './state/classifier.js';
import type { DerivedState, Intent } from './state/onboarding-state.js';
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
import {
getActiveProcessTrace,
ProcessTraceCollector,
runWithProcessTrace,
} from './runtime/trace.js';
import { validateOnboardingResponse } from './runtime/validator.js';
import { normalizeOnboardingSession } from './session.js';
import type { OnboardingSession, PromptContext, StoredOnboardingSession } from './types.js';
@@ -71,19 +81,57 @@ export class OnboardingSkill implements Skill {
async handle(
request: AgentRequest,
conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
if (getActiveProcessTrace()) {
return this.handleWithTrace(request, conversationHistory);
}
const collector = new ProcessTraceCollector();
return runWithProcessTrace(collector, () =>
this.handleWithTrace(request, conversationHistory),
);
}
private async handleWithTrace(
request: AgentRequest,
conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const logger = getRequestLogger();
const trace = getActiveProcessTrace();
const session = await this.loadSession(request.chatroomId, request.userId);
if (request.trigger === 'user-message' && !request.userMessage?.trim()) {
return this.makeEmptyResponse(request.requestId);
}
const classifyStep = trace?.startStep('classify');
const classified = await this.classifier.classify(request.userMessage, request.trigger);
classifyStep?.finish({
confidence: classified.confidence,
intent: classified.intent.kind,
source: classified.source,
});
const deriveStep = trace?.startStep('derive');
const derived = deriveState(session, request.trigger, classified.intent);
deriveStep?.finish({
phase: derived.phase.kind,
responseMode: derived.responseMode,
shouldRespond: derived.shouldRespond,
tier: derived.tier,
});
if (!derived.shouldRespond) {
return this.makeEmptyResponse(request.requestId);
const response = this.makeEmptyResponse(request.requestId);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse('[NO_RESPONSE]', derived, {
allowShortResponse: true,
});
return response;
}
let response: AgentResponse;
@@ -102,6 +150,15 @@ export class OnboardingSkill implements Skill {
}
}
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, {
allowShortResponse: derived.allowShortResponse,
});
if (derived.intent.kind === 'discord-verify-question') {
logger.debug(
{
@@ -122,6 +179,7 @@ export class OnboardingSkill implements Skill {
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
});
@@ -185,7 +243,9 @@ export class OnboardingSkill implements Skill {
userMessage: this.buildPromptUserMessage(request, derived),
};
return PromptBuilder.forDerived(derived, session, context).build();
const builder = PromptBuilder.forDerived(derived, session, context);
const promptVariantId = getPromptVariantId();
return promptVariantId ? builder.applyVariant(promptVariantId).build() : builder.build();
}
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
+1 -1
View File
@@ -1,7 +1,7 @@
import { DEADLINE_NEAR_THRESHOLD_MIN } from "../constants.js";
import { analyzeNicknameIssues } from "../onboarding-templates.js";
import type { OnboardingSession } from "../types.js";
import type { OnboardingPhase, RemainingStep } from "./onboarding-state.js";
import type { OnboardingPhase } from "./onboarding-state.js";
import {
getRemainingSteps,
isOnboardingCompleted,