import type { AgentRequest, AgentResponse, ConversationMessage, } from "@agent/types"; import type { Blackboard, OnboardingSession } from "@skills/onboarding/types"; import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state"; export interface EvalRoomContextMessage { senderId: string; nickname: string; message: string; timestamp: string; } export interface EvalCaseInput { request: AgentRequest; conversationHistory?: ConversationMessage[] | undefined; roomContext?: EvalRoomContextMessage[] | undefined; session?: OnboardingSession | null | undefined; blackboard?: Blackboard | null | undefined; summary?: string | undefined; } export interface EvalCaseExpected { intent?: string | undefined; mustIncludeAny?: string[] | undefined; mustNotInclude?: string[] | undefined; responseMode?: string | undefined; } export interface EvalCaseDefinition { id: string; lineNumber: number; name?: string | undefined; notes?: string | undefined; input: EvalCaseInput; expected: EvalCaseExpected; } export interface RuleJudgeCheck { name: | "validator" | "intent" | "responseMode" | "mustIncludeAny" | "mustNotInclude"; passed: boolean; skipped: boolean; detail?: string | undefined; } export interface RuleJudgeResult { passed: boolean; score: number; checks: RuleJudgeCheck[]; validatorStatus: "pass" | "soft-warn" | "reject"; validatorReasons: string[]; } export interface LlmJudgeScores { friendliness: number; accuracy: number; concision: number; scope: number; overall: number; } export type LlmJudgeResult = | { skipped: true; reason: string; raw?: string | undefined; } | { skipped: false; model: string; endpoint: string; provider: string; scores: LlmJudgeScores; rationale: string; raw: string; }; export interface EvalDerivedSnapshot { allowShortResponse: boolean; intent: string; phase: string; responseMode: DerivedState["responseMode"]; shouldRespond: boolean; suggestedMaxTokens: number; tier: DerivedState["tier"]; } export interface EvalClassifiedSnapshot { confidence: number; intent: string; source: ClassifiedIntent["source"]; } export interface EvalCaseResult { id: string; lineNumber: number; name?: string | undefined; latencyMs: number; classified: EvalClassifiedSnapshot; derived: EvalDerivedSnapshot; request: AgentRequest; response: AgentResponse; ruleJudge: RuleJudgeResult; llmJudge: LlmJudgeResult; error?: string | undefined; } export interface EvalReportSummary { averageLatencyMs: number; cases: number; emptyResponses: number; llmAverageAccuracy: number | null; llmAverageConcision: number | null; llmAverageFriendliness: number | null; llmAverageOverall: number | null; llmAverageScope: number | null; llmJudgedCases: number; passRate: number; passedCases: number; ruleAverageScore: number; } export interface EvalSummaryDelta { averageLatencyMs: number | null; llmAverageOverall: number | null; passRate: number | null; ruleAverageScore: number | null; } export interface EvalReport { baselineDelta?: EvalSummaryDelta | undefined; generatedAt: string; inputPath: string; setName: string; summary: EvalReportSummary; cases: EvalCaseResult[]; } export interface EvalBaselineFile { reportPath: string; setName: string; summary: EvalReportSummary; updatedAt: string; }