This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 1453433b62
commit da76af3826
65 files changed
+11089 -143

No files matched your search

+33
View File
@@ -5,6 +5,7 @@ import type {
} from "@agent/types";
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
export interface EvalRoomContextMessage {
senderId: string;
@@ -26,6 +27,12 @@ export interface EvalCaseExpected {
intent?: string | undefined;
mustIncludeAny?: string[] | undefined;
mustNotInclude?: string[] | undefined;
processExpected?: {
expectedApproach?: string | undefined;
expectedTools?: string[] | undefined;
minSteps?: number | undefined;
toolMode?: "exact" | "subset" | undefined;
} | undefined;
responseMode?: string | undefined;
}
@@ -82,6 +89,30 @@ export type LlmJudgeResult =
raw: string;
};
export interface ProcessJudgeCheck {
detail?: string | undefined;
passed: boolean;
rule:
| "critic-decision-justified"
| "intent-match"
| "min-steps"
| "planner-approach"
| "tier-appropriate"
| "tool-args-valid"
| "tools-selected-correctly"
| "validate-no-reject";
skipped: boolean;
step: string;
}
export interface ProcessJudgeResult {
checks: ProcessJudgeCheck[];
passedCount: number;
rationale?: string | undefined;
score: number;
totalCount: number;
}
export interface EvalDerivedSnapshot {
allowShortResponse: boolean;
intent: string;
@@ -109,6 +140,8 @@ export interface EvalCaseResult {
response: AgentResponse;
ruleJudge: RuleJudgeResult;
llmJudge: LlmJudgeResult;
processJudge?: ProcessJudgeResult | undefined;
processTrace?: ProcessTraceSnapshot | undefined;
error?: string | undefined;
}