Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -6,6 +6,7 @@ import {
|
||||
getTier2CriticModel,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface CritiqueResult {
|
||||
changed: boolean;
|
||||
@@ -55,6 +56,7 @@ export class OnboardingCritic {
|
||||
messages: ChatMessage[],
|
||||
options?: { deadlineMs?: number | undefined },
|
||||
): Promise<CritiqueResult> {
|
||||
const traceStep = getActiveProcessTrace()?.startStep("critic");
|
||||
const response = await this.llmClient.chatCompletion(
|
||||
buildCriticMessages(draft, derived, messages),
|
||||
{
|
||||
@@ -69,12 +71,20 @@ export class OnboardingCritic {
|
||||
|
||||
const text = response?.content.trim() ?? "";
|
||||
if (!text || text.startsWith("[OK]")) {
|
||||
traceStep?.finish({
|
||||
decision: "keep",
|
||||
rationale: text || "empty critic response",
|
||||
});
|
||||
return {
|
||||
changed: false,
|
||||
text: draft,
|
||||
};
|
||||
}
|
||||
|
||||
traceStep?.finish({
|
||||
decision: "rewrite",
|
||||
rationale: text.slice(0, 200),
|
||||
});
|
||||
return {
|
||||
changed: true,
|
||||
text,
|
||||
|
||||
@@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
import type { PlannerOutput } from "./planner-schema.js";
|
||||
import { PlannerOutputSchema } from "./planner-schema.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
|
||||
export interface OrchestratorRunOptions {
|
||||
@@ -260,6 +261,17 @@ export class OnboardingOrchestrator {
|
||||
messages: ChatMessage[],
|
||||
): Promise<PlannerOutput> {
|
||||
const logger = getRequestLogger();
|
||||
const traceStep = getActiveProcessTrace()?.startStep("plan");
|
||||
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
|
||||
traceStep?.finish({
|
||||
approach: plan.approach,
|
||||
mustAvoid: plan.mustAvoid,
|
||||
mustInclude: plan.mustInclude,
|
||||
suggestedTools: plan.suggestedTools,
|
||||
targetLengthChars: plan.targetLengthChars,
|
||||
});
|
||||
return plan;
|
||||
};
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
@@ -283,7 +295,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return firstPlan;
|
||||
return finishTrace(firstPlan);
|
||||
}
|
||||
|
||||
const retryMessages = [
|
||||
@@ -315,7 +327,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return retryPlan;
|
||||
return finishTrace(retryPlan);
|
||||
}
|
||||
|
||||
const fallbackPlan = getDefaultPlan(derived);
|
||||
@@ -327,7 +339,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner fell back to default plan",
|
||||
);
|
||||
return fallbackPlan;
|
||||
return finishTrace(fallbackPlan);
|
||||
}
|
||||
|
||||
private async runTier0(
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { parseToolCalls } from "@tools/parser";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
export interface ToolLoopOptions {
|
||||
deadlineMs?: number | undefined;
|
||||
@@ -107,15 +108,33 @@ export async function runWithTools(
|
||||
"[ToolLoop] Executing tool call",
|
||||
);
|
||||
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
const traceStep = getActiveProcessTrace()?.startStep("tool-call");
|
||||
let result: unknown;
|
||||
try {
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
name: tool.name,
|
||||
ok: true,
|
||||
resultSummary: summarizeToolResult(result),
|
||||
});
|
||||
} catch (error) {
|
||||
traceStep?.finish({
|
||||
args: call.arguments,
|
||||
name: tool.name,
|
||||
ok: false,
|
||||
resultSummary: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
@@ -137,3 +156,11 @@ export async function runWithTools(
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function summarizeToolResult(result: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(result).slice(0, 200);
|
||||
} catch {
|
||||
return String(result).slice(0, 200);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
|
||||
export type ProcessTraceStepName =
|
||||
| "classify"
|
||||
| "critic"
|
||||
| "derive"
|
||||
| "plan"
|
||||
| "respond"
|
||||
| "tool-call"
|
||||
| "validate";
|
||||
|
||||
export interface ProcessTraceStep<T> {
|
||||
name: ProcessTraceStepName;
|
||||
startedAtMs: number;
|
||||
durationMs: number;
|
||||
output: T;
|
||||
}
|
||||
|
||||
export interface ProcessTraceSnapshot {
|
||||
steps: {
|
||||
classify?: ProcessTraceStep<{
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: "regex" | "llm";
|
||||
}> | undefined;
|
||||
derive?: ProcessTraceStep<{
|
||||
phase: string;
|
||||
responseMode: string;
|
||||
shouldRespond: boolean;
|
||||
tier: 0 | 1 | 2;
|
||||
}> | undefined;
|
||||
plan?: ProcessTraceStep<{
|
||||
approach: string;
|
||||
mustAvoid: string[];
|
||||
mustInclude: string[];
|
||||
suggestedTools: string[];
|
||||
targetLengthChars: number;
|
||||
}> | undefined;
|
||||
toolCalls?: Array<
|
||||
ProcessTraceStep<{
|
||||
args: unknown;
|
||||
name: string;
|
||||
ok: boolean;
|
||||
resultSummary: string;
|
||||
}>
|
||||
> | undefined;
|
||||
respond?: ProcessTraceStep<{
|
||||
draftText: string;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
}> | undefined;
|
||||
critic?: ProcessTraceStep<{
|
||||
decision: "keep" | "rewrite";
|
||||
rationale: string;
|
||||
}> | undefined;
|
||||
validate?: ProcessTraceStep<{
|
||||
reasons: string[];
|
||||
status: "pass" | "reject" | "soft-warn";
|
||||
}> | undefined;
|
||||
};
|
||||
}
|
||||
|
||||
type ProcessTraceOutputByName = {
|
||||
classify: NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"];
|
||||
critic: NonNullable<ProcessTraceSnapshot["steps"]["critic"]>["output"];
|
||||
derive: NonNullable<ProcessTraceSnapshot["steps"]["derive"]>["output"];
|
||||
plan: NonNullable<ProcessTraceSnapshot["steps"]["plan"]>["output"];
|
||||
respond: NonNullable<ProcessTraceSnapshot["steps"]["respond"]>["output"];
|
||||
"tool-call": NonNullable<
|
||||
NonNullable<ProcessTraceSnapshot["steps"]["toolCalls"]>[number]
|
||||
>["output"];
|
||||
validate: NonNullable<ProcessTraceSnapshot["steps"]["validate"]>["output"];
|
||||
};
|
||||
|
||||
const processTraceStorage = new AsyncLocalStorage<ProcessTraceCollector>();
|
||||
|
||||
export class ProcessTraceCollector {
|
||||
private readonly steps: ProcessTraceSnapshot["steps"] = {};
|
||||
|
||||
startStep<Name extends ProcessTraceStepName>(
|
||||
name: Name,
|
||||
): {
|
||||
finish: (output: ProcessTraceOutputByName[Name]) => void;
|
||||
} {
|
||||
const startedAtMs = Date.now();
|
||||
let finished = false;
|
||||
|
||||
return {
|
||||
finish: (output: ProcessTraceOutputByName[Name]): void => {
|
||||
if (finished) {
|
||||
return;
|
||||
}
|
||||
finished = true;
|
||||
|
||||
const step: ProcessTraceStep<ProcessTraceOutputByName[Name]> = {
|
||||
name,
|
||||
startedAtMs,
|
||||
durationMs: Math.max(0, Date.now() - startedAtMs),
|
||||
output,
|
||||
};
|
||||
|
||||
if (name === "tool-call") {
|
||||
this.steps.toolCalls = [
|
||||
...(this.steps.toolCalls ?? []),
|
||||
step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["tool-call"]
|
||||
>,
|
||||
];
|
||||
return;
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case "classify":
|
||||
this.steps.classify = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["classify"]
|
||||
>;
|
||||
return;
|
||||
case "critic":
|
||||
this.steps.critic = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["critic"]
|
||||
>;
|
||||
return;
|
||||
case "derive":
|
||||
this.steps.derive = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["derive"]
|
||||
>;
|
||||
return;
|
||||
case "plan":
|
||||
this.steps.plan = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["plan"]
|
||||
>;
|
||||
return;
|
||||
case "respond":
|
||||
this.steps.respond = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["respond"]
|
||||
>;
|
||||
return;
|
||||
case "validate":
|
||||
this.steps.validate = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["validate"]
|
||||
>;
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): ProcessTraceSnapshot {
|
||||
return {
|
||||
steps: {
|
||||
...this.steps,
|
||||
toolCalls: this.steps.toolCalls
|
||||
? [...this.steps.toolCalls]
|
||||
: undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function runWithProcessTrace<T>(
|
||||
collector: ProcessTraceCollector,
|
||||
fn: () => T,
|
||||
): T {
|
||||
return processTraceStorage.run(collector, fn);
|
||||
}
|
||||
|
||||
export function getActiveProcessTrace(): ProcessTraceCollector | undefined {
|
||||
return processTraceStorage.getStore();
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { getMaxResponseLength } from "../../../config.js";
|
||||
import { VALID_TIER_SET } from "../constants.js";
|
||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
|
||||
const ALLOWED_URL_PATTERNS = [
|
||||
/discord\.gg\/lolympus/iu,
|
||||
@@ -118,6 +119,10 @@ function markValidation(result: ValidationResult): ValidationResult {
|
||||
"agent.validator.status": result.status,
|
||||
"agent.validator.warnings": result.reasons,
|
||||
});
|
||||
getActiveProcessTrace()?.startStep("validate").finish({
|
||||
status: result.status,
|
||||
reasons: result.reasons,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user