Add evaluation operations and reporting

This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 60f33b8f4b
commit a900016be4
60 files changed
+8689 -142

No files matched your search

+15 -3
View File
@@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js";
import { OnboardingCritic } from "./critic.js";
import type { PlannerOutput } from "./planner-schema.js";
import { PlannerOutputSchema } from "./planner-schema.js";
import { getActiveProcessTrace } from "./trace.js";
import { runWithTools } from "./tool-loop.js";
export interface OrchestratorRunOptions {
@@ -260,6 +261,17 @@ export class OnboardingOrchestrator {
messages: ChatMessage[],
): Promise<PlannerOutput> {
const logger = getRequestLogger();
const traceStep = getActiveProcessTrace()?.startStep("plan");
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
traceStep?.finish({
approach: plan.approach,
mustAvoid: plan.mustAvoid,
mustInclude: plan.mustInclude,
suggestedTools: plan.suggestedTools,
targetLengthChars: plan.targetLengthChars,
});
return plan;
};
const plannerMessages = buildPlannerMessages(derived, messages);
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
deadlineMs: getRemainingDeadlineMs(),
@@ -283,7 +295,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner output parsed",
);
return firstPlan;
return finishTrace(firstPlan);
}
const retryMessages = [
@@ -315,7 +327,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner output parsed",
);
return retryPlan;
return finishTrace(retryPlan);
}
const fallbackPlan = getDefaultPlan(derived);
@@ -327,7 +339,7 @@ export class OnboardingOrchestrator {
},
"[OnboardingOrchestrator] Planner fell back to default plan",
);
return fallbackPlan;
return finishTrace(fallbackPlan);
}
private async runTier0(