Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit a900016be4
60 files changed
+8689 -142

No files matched your search

+10
View File
@@ -6,6 +6,7 @@ import {
getTier2CriticModel,
} from "../../../config.js";
import type { DerivedState } from "../state/onboarding-state.js";
import { getActiveProcessTrace } from "./trace.js";
export interface CritiqueResult {
changed: boolean;
@@ -55,6 +56,7 @@ export class OnboardingCritic {
messages: ChatMessage[],
options?: { deadlineMs?: number | undefined },
): Promise<CritiqueResult> {
const traceStep = getActiveProcessTrace()?.startStep("critic");
const response = await this.llmClient.chatCompletion(
buildCriticMessages(draft, derived, messages),
{
@@ -69,12 +71,20 @@ export class OnboardingCritic {
const text = response?.content.trim() ?? "";
if (!text || text.startsWith("[OK]")) {
traceStep?.finish({
decision: "keep",
rationale: text || "empty critic response",
});
return {
changed: false,
text: draft,
};
}
traceStep?.finish({
decision: "rewrite",
rationale: text.slice(0, 200),
});
return {
changed: true,
text,
+15 -3
View File
@@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js";
import { OnboardingCritic } from "./critic.js";
import type { PlannerOutput } from "./planner-schema.js";
import { PlannerOutputSchema } from "./planner-schema.js";
import { getActiveProcessTrace } from "./trace.js";
import { runWithTools } from "./tool-loop.js";
export interface OrchestratorRunOptions {
@@ -260,6 +261,17 @@ export class OnboardingOrchestrator {
messages: ChatMessage[],
): Promise<PlannerOutput> {
const logger = getRequestLogger();
const traceStep = getActiveProcessTrace()?.startStep("plan");
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
traceStep?.finish({
approach: plan.approach,
mustAvoid: plan.mustAvoid,
mustInclude: plan.mustInclude,
suggestedTools: plan.suggestedTools,
targetLengthChars: plan.targetLengthChars,
});
return plan;
};
const plannerMessages = buildPlannerMessages(derived, messages);
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
deadlineMs: getRemainingDeadlineMs(),
@@ -283,7 +295,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner output parsed",
);
return firstPlan;
return finishTrace(firstPlan);
}
const retryMessages = [
@@ -315,7 +327,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner output parsed",
);
return retryPlan;
return finishTrace(retryPlan);
}
const fallbackPlan = getDefaultPlan(derived);
@@ -327,7 +339,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner fell back to default plan",
);
return fallbackPlan;
return finishTrace(fallbackPlan);
}
private async runTier0(
+36 -9
View File
@@ -4,6 +4,7 @@ import type { LlmClient } from "@llm/llm-client";
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
import { parseToolCalls } from "@tools/parser";
import type { Tool, ToolContext } from "@tools/tool";
import { getActiveProcessTrace } from "./trace.js";
export interface ToolLoopOptions {
deadlineMs?: number | undefined;
@@ -107,15 +108,33 @@ export async function runWithTools(
"[ToolLoop] Executing tool call",
);
const validated = tool.argsSchema.parse(call.arguments);
const result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
let result: unknown;
try {
const validated = tool.argsSchema.parse(call.arguments);
result = await withToolSpan(
tool.name,
{
"tool.call.id": call.id,
"tool.name": tool.name,
},
() => tool.execute(validated, options.toolContext),
);
traceStep?.finish({
args: call.arguments,
name: tool.name,
ok: true,
resultSummary: summarizeToolResult(result),
});
} catch (error) {
traceStep?.finish({
args: call.arguments,
name: tool.name,
ok: false,
resultSummary: error instanceof Error ? error.message : String(error),
});
throw error;
}
logger.debug(
{
@@ -137,3 +156,11 @@ export async function runWithTools(
return null;
}
function summarizeToolResult(result: unknown): string {
try {
return JSON.stringify(result).slice(0, 200);
} catch {
return String(result).slice(0, 200);
}
}
+169
View File
@@ -0,0 +1,169 @@
import { AsyncLocalStorage } from "node:async_hooks";
export type ProcessTraceStepName =
| "classify"
| "critic"
| "derive"
| "plan"
| "respond"
| "tool-call"
| "validate";
export interface ProcessTraceStep<T> {
name: ProcessTraceStepName;
startedAtMs: number;
durationMs: number;
output: T;
}
export interface ProcessTraceSnapshot {
steps: {
classify?: ProcessTraceStep<{
confidence: number;
intent: string;
source: "regex" | "llm";
}> | undefined;
derive?: ProcessTraceStep<{
phase: string;
responseMode: string;
shouldRespond: boolean;
tier: 0 | 1 | 2;
}> | undefined;
plan?: ProcessTraceStep<{
approach: string;
mustAvoid: string[];
mustInclude: string[];
suggestedTools: string[];
targetLengthChars: number;
}> | undefined;
toolCalls?: Array<
ProcessTraceStep<{
args: unknown;
name: string;
ok: boolean;
resultSummary: string;
}>
> | undefined;
respond?: ProcessTraceStep<{
draftText: string;
tokensIn: number;
tokensOut: number;
}> | undefined;
critic?: ProcessTraceStep<{
decision: "keep" | "rewrite";
rationale: string;
}> | undefined;
validate?: ProcessTraceStep<{
reasons: string[];
status: "pass" | "reject" | "soft-warn";
}> | undefined;
};
}
type ProcessTraceOutputByName = {
classify: NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"];
critic: NonNullable<ProcessTraceSnapshot["steps"]["critic"]>["output"];
derive: NonNullable<ProcessTraceSnapshot["steps"]["derive"]>["output"];
plan: NonNullable<ProcessTraceSnapshot["steps"]["plan"]>["output"];
respond: NonNullable<ProcessTraceSnapshot["steps"]["respond"]>["output"];
"tool-call": NonNullable<
NonNullable<ProcessTraceSnapshot["steps"]["toolCalls"]>[number]
>["output"];
validate: NonNullable<ProcessTraceSnapshot["steps"]["validate"]>["output"];
};
const processTraceStorage = new AsyncLocalStorage<ProcessTraceCollector>();
export class ProcessTraceCollector {
private readonly steps: ProcessTraceSnapshot["steps"] = {};
startStep<Name extends ProcessTraceStepName>(
name: Name,
): {
finish: (output: ProcessTraceOutputByName[Name]) => void;
} {
const startedAtMs = Date.now();
let finished = false;
return {
finish: (output: ProcessTraceOutputByName[Name]): void => {
if (finished) {
return;
}
finished = true;
const step: ProcessTraceStep<ProcessTraceOutputByName[Name]> = {
name,
startedAtMs,
durationMs: Math.max(0, Date.now() - startedAtMs),
output,
};
if (name === "tool-call") {
this.steps.toolCalls = [
...(this.steps.toolCalls ?? []),
step as ProcessTraceStep<
ProcessTraceOutputByName["tool-call"]
>,
];
return;
}
switch (name) {
case "classify":
this.steps.classify = step as ProcessTraceStep<
ProcessTraceOutputByName["classify"]
>;
return;
case "critic":
this.steps.critic = step as ProcessTraceStep<
ProcessTraceOutputByName["critic"]
>;
return;
case "derive":
this.steps.derive = step as ProcessTraceStep<
ProcessTraceOutputByName["derive"]
>;
return;
case "plan":
this.steps.plan = step as ProcessTraceStep<
ProcessTraceOutputByName["plan"]
>;
return;
case "respond":
this.steps.respond = step as ProcessTraceStep<
ProcessTraceOutputByName["respond"]
>;
return;
case "validate":
this.steps.validate = step as ProcessTraceStep<
ProcessTraceOutputByName["validate"]
>;
return;
}
},
};
}
snapshot(): ProcessTraceSnapshot {
return {
steps: {
...this.steps,
toolCalls: this.steps.toolCalls
? [...this.steps.toolCalls]
: undefined,
},
};
}
}
export function runWithProcessTrace<T>(
collector: ProcessTraceCollector,
fn: () => T,
): T {
return processTraceStorage.run(collector, fn);
}
export function getActiveProcessTrace(): ProcessTraceCollector | undefined {
return processTraceStorage.getStore();
}
@@ -3,6 +3,7 @@ import { getMaxResponseLength } from "../../../config.js";
import { VALID_TIER_SET } from "../constants.js";
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
import type { DerivedState } from "../state/onboarding-state.js";
import { getActiveProcessTrace } from "./trace.js";
const ALLOWED_URL_PATTERNS = [
/discord\.gg\/lolympus/iu,
@@ -118,6 +119,10 @@ function markValidation(result: ValidationResult): ValidationResult {
"agent.validator.status": result.status,
"agent.validator.warnings": result.reasons,
});
getActiveProcessTrace()?.startStep("validate").finish({
status: result.status,
reasons: result.reasons,
});
return result;
}