Files
lolympus-agent/src/eval/types.ts
T

152 lines
3.4 KiB
TypeScript

import type {
AgentRequest,
AgentResponse,
ConversationMessage,
} from "@agent/types";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
export interface EvalRoomContextMessage {
senderId: string;
nickname: string;
message: string;
timestamp: string;
}
export interface EvalCaseInput {
request: AgentRequest;
conversationHistory?: ConversationMessage[] | undefined;
roomContext?: EvalRoomContextMessage[] | undefined;
session?: OnboardingSession | null | undefined;
blackboard?: Blackboard | null | undefined;
summary?: string | undefined;
}
export interface EvalCaseExpected {
intent?: string | undefined;
mustIncludeAny?: string[] | undefined;
mustNotInclude?: string[] | undefined;
responseMode?: string | undefined;
}
export interface EvalCaseDefinition {
id: string;
lineNumber: number;
name?: string | undefined;
notes?: string | undefined;
input: EvalCaseInput;
expected: EvalCaseExpected;
}
export interface RuleJudgeCheck {
name:
| "validator"
| "intent"
| "responseMode"
| "mustIncludeAny"
| "mustNotInclude";
passed: boolean;
skipped: boolean;
detail?: string | undefined;
}
export interface RuleJudgeResult {
passed: boolean;
score: number;
checks: RuleJudgeCheck[];
validatorStatus: "pass" | "soft-warn" | "reject";
validatorReasons: string[];
}
export interface LlmJudgeScores {
friendliness: number;
accuracy: number;
concision: number;
scope: number;
overall: number;
}
export type LlmJudgeResult =
| {
skipped: true;
reason: string;
raw?: string | undefined;
}
| {
skipped: false;
model: string;
endpoint: string;
provider: string;
scores: LlmJudgeScores;
rationale: string;
raw: string;
};
export interface EvalDerivedSnapshot {
allowShortResponse: boolean;
intent: string;
phase: string;
responseMode: DerivedState["responseMode"];
shouldRespond: boolean;
suggestedMaxTokens: number;
tier: DerivedState["tier"];
}
export interface EvalClassifiedSnapshot {
confidence: number;
intent: string;
source: ClassifiedIntent["source"];
}
export interface EvalCaseResult {
id: string;
lineNumber: number;
name?: string | undefined;
latencyMs: number;
classified: EvalClassifiedSnapshot;
derived: EvalDerivedSnapshot;
request: AgentRequest;
response: AgentResponse;
ruleJudge: RuleJudgeResult;
llmJudge: LlmJudgeResult;
error?: string | undefined;
}
export interface EvalReportSummary {
averageLatencyMs: number;
cases: number;
emptyResponses: number;
llmAverageAccuracy: number | null;
llmAverageConcision: number | null;
llmAverageFriendliness: number | null;
llmAverageOverall: number | null;
llmAverageScope: number | null;
llmJudgedCases: number;
passRate: number;
passedCases: number;
ruleAverageScore: number;
}
export interface EvalSummaryDelta {
averageLatencyMs: number | null;
llmAverageOverall: number | null;
passRate: number | null;
ruleAverageScore: number | null;
}
export interface EvalReport {
baselineDelta?: EvalSummaryDelta | undefined;
generatedAt: string;
inputPath: string;
setName: string;
summary: EvalReportSummary;
cases: EvalCaseResult[];
}
export interface EvalBaselineFile {
reportPath: string;
setName: string;
summary: EvalReportSummary;
updatedAt: string;
}