Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -19,6 +19,7 @@ import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
import type { PlannerOutput } from "./planner-schema.js";
|
||||
import { PlannerOutputSchema } from "./planner-schema.js";
|
||||
import { getActiveProcessTrace } from "./trace.js";
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
|
||||
export interface OrchestratorRunOptions {
|
||||
@@ -260,6 +261,17 @@ export class OnboardingOrchestrator {
|
||||
messages: ChatMessage[],
|
||||
): Promise<PlannerOutput> {
|
||||
const logger = getRequestLogger();
|
||||
const traceStep = getActiveProcessTrace()?.startStep("plan");
|
||||
const finishTrace = (plan: PlannerOutput): PlannerOutput => {
|
||||
traceStep?.finish({
|
||||
approach: plan.approach,
|
||||
mustAvoid: plan.mustAvoid,
|
||||
mustInclude: plan.mustInclude,
|
||||
suggestedTools: plan.suggestedTools,
|
||||
targetLengthChars: plan.targetLengthChars,
|
||||
});
|
||||
return plan;
|
||||
};
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
@@ -283,7 +295,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return firstPlan;
|
||||
return finishTrace(firstPlan);
|
||||
}
|
||||
|
||||
const retryMessages = [
|
||||
@@ -315,7 +327,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner output parsed",
|
||||
);
|
||||
return retryPlan;
|
||||
return finishTrace(retryPlan);
|
||||
}
|
||||
|
||||
const fallbackPlan = getDefaultPlan(derived);
|
||||
@@ -327,7 +339,7 @@ export class OnboardingOrchestrator {
|
||||
},
|
||||
"[OnboardingOrchestrator] Planner fell back to default plan",
|
||||
);
|
||||
return fallbackPlan;
|
||||
return finishTrace(fallbackPlan);
|
||||
}
|
||||
|
||||
private async runTier0(
|
||||
|
||||
Reference in new issue
Block a user