152 lines
3.4 KiB
TypeScript
152 lines
3.4 KiB
TypeScript
import type {
|
|
AgentRequest,
|
|
AgentResponse,
|
|
ConversationMessage,
|
|
} from "@agent/types";
|
|
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
|
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
|
|
|
export interface EvalRoomContextMessage {
|
|
senderId: string;
|
|
nickname: string;
|
|
message: string;
|
|
timestamp: string;
|
|
}
|
|
|
|
export interface EvalCaseInput {
|
|
request: AgentRequest;
|
|
conversationHistory?: ConversationMessage[] | undefined;
|
|
roomContext?: EvalRoomContextMessage[] | undefined;
|
|
session?: OnboardingSession | null | undefined;
|
|
blackboard?: Blackboard | null | undefined;
|
|
summary?: string | undefined;
|
|
}
|
|
|
|
export interface EvalCaseExpected {
|
|
intent?: string | undefined;
|
|
mustIncludeAny?: string[] | undefined;
|
|
mustNotInclude?: string[] | undefined;
|
|
responseMode?: string | undefined;
|
|
}
|
|
|
|
export interface EvalCaseDefinition {
|
|
id: string;
|
|
lineNumber: number;
|
|
name?: string | undefined;
|
|
notes?: string | undefined;
|
|
input: EvalCaseInput;
|
|
expected: EvalCaseExpected;
|
|
}
|
|
|
|
export interface RuleJudgeCheck {
|
|
name:
|
|
| "validator"
|
|
| "intent"
|
|
| "responseMode"
|
|
| "mustIncludeAny"
|
|
| "mustNotInclude";
|
|
passed: boolean;
|
|
skipped: boolean;
|
|
detail?: string | undefined;
|
|
}
|
|
|
|
export interface RuleJudgeResult {
|
|
passed: boolean;
|
|
score: number;
|
|
checks: RuleJudgeCheck[];
|
|
validatorStatus: "pass" | "soft-warn" | "reject";
|
|
validatorReasons: string[];
|
|
}
|
|
|
|
export interface LlmJudgeScores {
|
|
friendliness: number;
|
|
accuracy: number;
|
|
concision: number;
|
|
scope: number;
|
|
overall: number;
|
|
}
|
|
|
|
export type LlmJudgeResult =
|
|
| {
|
|
skipped: true;
|
|
reason: string;
|
|
raw?: string | undefined;
|
|
}
|
|
| {
|
|
skipped: false;
|
|
model: string;
|
|
endpoint: string;
|
|
provider: string;
|
|
scores: LlmJudgeScores;
|
|
rationale: string;
|
|
raw: string;
|
|
};
|
|
|
|
export interface EvalDerivedSnapshot {
|
|
allowShortResponse: boolean;
|
|
intent: string;
|
|
phase: string;
|
|
responseMode: DerivedState["responseMode"];
|
|
shouldRespond: boolean;
|
|
suggestedMaxTokens: number;
|
|
tier: DerivedState["tier"];
|
|
}
|
|
|
|
export interface EvalClassifiedSnapshot {
|
|
confidence: number;
|
|
intent: string;
|
|
source: ClassifiedIntent["source"];
|
|
}
|
|
|
|
export interface EvalCaseResult {
|
|
id: string;
|
|
lineNumber: number;
|
|
name?: string | undefined;
|
|
latencyMs: number;
|
|
classified: EvalClassifiedSnapshot;
|
|
derived: EvalDerivedSnapshot;
|
|
request: AgentRequest;
|
|
response: AgentResponse;
|
|
ruleJudge: RuleJudgeResult;
|
|
llmJudge: LlmJudgeResult;
|
|
error?: string | undefined;
|
|
}
|
|
|
|
export interface EvalReportSummary {
|
|
averageLatencyMs: number;
|
|
cases: number;
|
|
emptyResponses: number;
|
|
llmAverageAccuracy: number | null;
|
|
llmAverageConcision: number | null;
|
|
llmAverageFriendliness: number | null;
|
|
llmAverageOverall: number | null;
|
|
llmAverageScope: number | null;
|
|
llmJudgedCases: number;
|
|
passRate: number;
|
|
passedCases: number;
|
|
ruleAverageScore: number;
|
|
}
|
|
|
|
export interface EvalSummaryDelta {
|
|
averageLatencyMs: number | null;
|
|
llmAverageOverall: number | null;
|
|
passRate: number | null;
|
|
ruleAverageScore: number | null;
|
|
}
|
|
|
|
export interface EvalReport {
|
|
baselineDelta?: EvalSummaryDelta | undefined;
|
|
generatedAt: string;
|
|
inputPath: string;
|
|
setName: string;
|
|
summary: EvalReportSummary;
|
|
cases: EvalCaseResult[];
|
|
}
|
|
|
|
export interface EvalBaselineFile {
|
|
reportPath: string;
|
|
setName: string;
|
|
summary: EvalReportSummary;
|
|
updatedAt: string;
|
|
}
|