Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
@@ -0,0 +1,151 @@
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
export interface EvalRoomContextMessage {
|
||||
senderId: string;
|
||||
nickname: string;
|
||||
message: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface EvalCaseInput {
|
||||
request: AgentRequest;
|
||||
conversationHistory?: ConversationMessage[] | undefined;
|
||||
roomContext?: EvalRoomContextMessage[] | undefined;
|
||||
session?: OnboardingSession | null | undefined;
|
||||
blackboard?: Blackboard | null | undefined;
|
||||
summary?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalCaseExpected {
|
||||
intent?: string | undefined;
|
||||
mustIncludeAny?: string[] | undefined;
|
||||
mustNotInclude?: string[] | undefined;
|
||||
responseMode?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalCaseDefinition {
|
||||
id: string;
|
||||
lineNumber: number;
|
||||
name?: string | undefined;
|
||||
notes?: string | undefined;
|
||||
input: EvalCaseInput;
|
||||
expected: EvalCaseExpected;
|
||||
}
|
||||
|
||||
export interface RuleJudgeCheck {
|
||||
name:
|
||||
| "validator"
|
||||
| "intent"
|
||||
| "responseMode"
|
||||
| "mustIncludeAny"
|
||||
| "mustNotInclude";
|
||||
passed: boolean;
|
||||
skipped: boolean;
|
||||
detail?: string | undefined;
|
||||
}
|
||||
|
||||
export interface RuleJudgeResult {
|
||||
passed: boolean;
|
||||
score: number;
|
||||
checks: RuleJudgeCheck[];
|
||||
validatorStatus: "pass" | "soft-warn" | "reject";
|
||||
validatorReasons: string[];
|
||||
}
|
||||
|
||||
export interface LlmJudgeScores {
|
||||
friendliness: number;
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
scope: number;
|
||||
overall: number;
|
||||
}
|
||||
|
||||
export type LlmJudgeResult =
|
||||
| {
|
||||
skipped: true;
|
||||
reason: string;
|
||||
raw?: string | undefined;
|
||||
}
|
||||
| {
|
||||
skipped: false;
|
||||
model: string;
|
||||
endpoint: string;
|
||||
provider: string;
|
||||
scores: LlmJudgeScores;
|
||||
rationale: string;
|
||||
raw: string;
|
||||
};
|
||||
|
||||
export interface EvalDerivedSnapshot {
|
||||
allowShortResponse: boolean;
|
||||
intent: string;
|
||||
phase: string;
|
||||
responseMode: DerivedState["responseMode"];
|
||||
shouldRespond: boolean;
|
||||
suggestedMaxTokens: number;
|
||||
tier: DerivedState["tier"];
|
||||
}
|
||||
|
||||
export interface EvalClassifiedSnapshot {
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: ClassifiedIntent["source"];
|
||||
}
|
||||
|
||||
export interface EvalCaseResult {
|
||||
id: string;
|
||||
lineNumber: number;
|
||||
name?: string | undefined;
|
||||
latencyMs: number;
|
||||
classified: EvalClassifiedSnapshot;
|
||||
derived: EvalDerivedSnapshot;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
ruleJudge: RuleJudgeResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
error?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalReportSummary {
|
||||
averageLatencyMs: number;
|
||||
cases: number;
|
||||
emptyResponses: number;
|
||||
llmAverageAccuracy: number | null;
|
||||
llmAverageConcision: number | null;
|
||||
llmAverageFriendliness: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
llmAverageScope: number | null;
|
||||
llmJudgedCases: number;
|
||||
passRate: number;
|
||||
passedCases: number;
|
||||
ruleAverageScore: number;
|
||||
}
|
||||
|
||||
export interface EvalSummaryDelta {
|
||||
averageLatencyMs: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
}
|
||||
|
||||
export interface EvalReport {
|
||||
baselineDelta?: EvalSummaryDelta | undefined;
|
||||
generatedAt: string;
|
||||
inputPath: string;
|
||||
setName: string;
|
||||
summary: EvalReportSummary;
|
||||
cases: EvalCaseResult[];
|
||||
}
|
||||
|
||||
export interface EvalBaselineFile {
|
||||
reportPath: string;
|
||||
setName: string;
|
||||
summary: EvalReportSummary;
|
||||
updatedAt: string;
|
||||
}
|
||||
Reference in new issue
Block a user