Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -5,6 +5,7 @@ import type {
|
||||
} from "@agent/types";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
|
||||
export interface EvalRoomContextMessage {
|
||||
senderId: string;
|
||||
@@ -26,6 +27,12 @@ export interface EvalCaseExpected {
|
||||
intent?: string | undefined;
|
||||
mustIncludeAny?: string[] | undefined;
|
||||
mustNotInclude?: string[] | undefined;
|
||||
processExpected?: {
|
||||
expectedApproach?: string | undefined;
|
||||
expectedTools?: string[] | undefined;
|
||||
minSteps?: number | undefined;
|
||||
toolMode?: "exact" | "subset" | undefined;
|
||||
} | undefined;
|
||||
responseMode?: string | undefined;
|
||||
}
|
||||
|
||||
@@ -82,6 +89,30 @@ export type LlmJudgeResult =
|
||||
raw: string;
|
||||
};
|
||||
|
||||
export interface ProcessJudgeCheck {
|
||||
detail?: string | undefined;
|
||||
passed: boolean;
|
||||
rule:
|
||||
| "critic-decision-justified"
|
||||
| "intent-match"
|
||||
| "min-steps"
|
||||
| "planner-approach"
|
||||
| "tier-appropriate"
|
||||
| "tool-args-valid"
|
||||
| "tools-selected-correctly"
|
||||
| "validate-no-reject";
|
||||
skipped: boolean;
|
||||
step: string;
|
||||
}
|
||||
|
||||
export interface ProcessJudgeResult {
|
||||
checks: ProcessJudgeCheck[];
|
||||
passedCount: number;
|
||||
rationale?: string | undefined;
|
||||
score: number;
|
||||
totalCount: number;
|
||||
}
|
||||
|
||||
export interface EvalDerivedSnapshot {
|
||||
allowShortResponse: boolean;
|
||||
intent: string;
|
||||
@@ -109,6 +140,8 @@ export interface EvalCaseResult {
|
||||
response: AgentResponse;
|
||||
ruleJudge: RuleJudgeResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
processJudge?: ProcessJudgeResult | undefined;
|
||||
processTrace?: ProcessTraceSnapshot | undefined;
|
||||
error?: string | undefined;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user