import type { AgentRequest, AgentResponse, ConversationMessage, } from "@agent/types"; import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state"; import type { ProcessTraceSnapshot, ProcessTraceStepName, } from "@skills/onboarding/runtime/trace"; import type { PrmFeatureVector } from "./judges/prm.js"; export interface EvalRoomContextMessage { senderId: string; nickname: string; message: string; timestamp: string; } export interface EvalCaseInput { request: AgentRequest; conversationHistory?: ConversationMessage[] | undefined; roomContext?: EvalRoomContextMessage[] | undefined; session?: OnboardingSession | null | undefined; blackboard?: Blackboard | null | undefined; summary?: string | undefined; } export interface EvalCaseExpected { intent?: string | undefined; mustIncludeAny?: string[] | undefined; mustNotInclude?: string[] | undefined; processExpected?: { expectedApproach?: string | undefined; expectedTools?: string[] | undefined; minSteps?: number | undefined; toolMode?: "exact" | "subset" | undefined; } | undefined; responseMode?: string | undefined; } export interface EvalCaseDefinition { id: string; lineNumber: number; name?: string | undefined; notes?: string | undefined; input: EvalCaseInput; expected: EvalCaseExpected; } export interface RuleJudgeCheck { name: | "validator" | "intent" | "responseMode" | "mustIncludeAny" | "mustNotInclude"; passed: boolean; skipped: boolean; detail?: string | undefined; } export interface RuleJudgeResult { passed: boolean; score: number; checks: RuleJudgeCheck[]; validatorStatus: "pass" | "soft-warn" | "reject"; validatorReasons: string[]; } export interface LlmJudgeScores { friendliness: number; accuracy: number; concision: number; scope: number; overall: number; } export type LlmJudgeResult = | { skipped: true; reason: string; raw?: string | undefined; } | { skipped: false; model: string; endpoint: string; provider: string; scores: LlmJudgeScores; rationale: string; raw: string; }; export interface ProcessJudgeCheck { detail?: string | undefined; passed: boolean; rule: | "critic-decision-justified" | "intent-match" | "min-steps" | "planner-approach" | "tier-appropriate" | "tool-args-valid" | "tool-plan-consistent" | "tools-selected-correctly" | "validate-no-reject"; skipped: boolean; step: string; } export interface ProcessJudgeResult { checks: ProcessJudgeCheck[]; passedCount: number; prmFeatures?: PrmFeatureVector | undefined; prmScore?: number | undefined; rationale?: string | undefined; rootCauseScoreLoss?: number | undefined; rootCauseStep?: ProcessTraceStepName | undefined; score: number; totalCount: number; } export interface EvalDerivedSnapshot { abstentionScore?: number | undefined; allowShortResponse: boolean; intent: string; phase: string; responseMode: DerivedState["responseMode"]; routingDecision?: DerivedState["routingDecision"] | undefined; shouldRespond: boolean; suggestedMaxTokens: number; tier: DerivedState["tier"]; } export interface EvalClassifiedSnapshot { confidence: number; intent: string; source: ClassifiedIntent["source"]; } export interface EvalCaseResult { id: string; lineNumber: number; name?: string | undefined; latencyMs: number; classified: EvalClassifiedSnapshot; derived: EvalDerivedSnapshot; request: AgentRequest; response: AgentResponse; ruleJudge: RuleJudgeResult; llmJudge: LlmJudgeResult; processJudge?: ProcessJudgeResult | undefined; processTrace?: ProcessTraceSnapshot | undefined; error?: string | undefined; } export interface EvalReportSummary { averageLatencyMs: number; cases: number; emptyResponses: number; llmAverageAccuracy: number | null; llmAverageConcision: number | null; llmAverageFriendliness: number | null; llmAverageOverall: number | null; llmAverageScope: number | null; llmJudgedCases: number; passRate: number; passedCases: number; ruleAverageScore: number; } export interface EvalSummaryDelta { averageLatencyMs: number | null; llmAverageOverall: number | null; passRate: number | null; ruleAverageScore: number | null; } export interface EvalReport { baselineDelta?: EvalSummaryDelta | undefined; generatedAt: string; inputPath: string; setName: string; summary: EvalReportSummary; cases: EvalCaseResult[]; } export interface EvalBaselineFile { reportPath: string; setName: string; summary: EvalReportSummary; updatedAt: string; }