Make eval pipeline skill-aware and validate SKILL manifests

This commit is contained in:
dahyeon committed 2026-05-18 16:26:31 -07:00
1 parent 12e731a428
commit 54bc295ead
35 files changed
+1280 -499

No files matched your search

+14 -11
View File
@@ -2,8 +2,7 @@ import { z } from "zod";
import type { AgentRequest } from "@agent/types";
import type { LlmClient } from "@llm/llm-client";
import { createBuiltInToolRegistry } from "@tools/builtins/index";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import type { ToolRegistry } from "@tools/registry";
import {
getEvalProcessLlmJudgeMaxTokens,
getEvalPrmEnabled,
@@ -12,6 +11,7 @@ import {
} from "../../config.js";
import type {
EvalCaseExpected,
EvalProcessTraceSnapshot,
ProcessJudgeCheck,
ProcessJudgeResult,
} from "../types.js";
@@ -80,7 +80,7 @@ function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudge
function enrichWithPrm(
result: ProcessJudgeResult,
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
): ProcessJudgeResult {
if (!getEvalPrmEnabled()) {
return result;
@@ -125,7 +125,7 @@ function hasAllTools(
return subset && actual.size === expected.size;
}
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
function countRecordedSteps(trace: EvalProcessTraceSnapshot): number {
const scalarSteps = [
trace.steps.classify,
trace.steps.derive,
@@ -138,18 +138,20 @@ function countRecordedSteps(trace: ProcessTraceSnapshot): number {
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
}
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
function validateToolArgs(
trace: EvalProcessTraceSnapshot,
toolRegistry: ToolRegistry,
): ProcessJudgeCheck {
const toolCalls = trace.steps.toolCalls ?? [];
if (toolCalls.length === 0) {
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
}
const registry = createBuiltInToolRegistry();
const failures: string[] = [];
for (const call of toolCalls) {
try {
registry.get(call.output.name).argsSchema.parse(call.output.args);
toolRegistry.get(call.output.name).argsSchema.parse(call.output.args);
} catch {
failures.push(call.output.name);
}
@@ -167,7 +169,8 @@ function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
export function judgeProcessRule(
expected: EvalCaseExpected,
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
toolRegistry: ToolRegistry,
): ProcessJudgeResult {
const checks: ProcessJudgeCheck[] = [];
@@ -244,7 +247,7 @@ export function judgeProcessRule(
);
}
checks.push(validateToolArgs(trace));
checks.push(validateToolArgs(trace, toolRegistry));
const toolPlanStepCount = trace.steps.plan?.output.toolPlanStepCount;
if (toolPlanStepCount !== undefined) {
@@ -318,7 +321,7 @@ export function judgeProcessRule(
}
function buildProcessJudgePrompt(
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
options: { finalText: string; request: AgentRequest },
): string {
return [
@@ -341,7 +344,7 @@ function buildProcessJudgePrompt(
export async function judgeProcessWithLlm(
llmClient: LlmClient,
trace: ProcessTraceSnapshot,
trace: EvalProcessTraceSnapshot,
options: { finalText: string; request: AgentRequest },
): Promise<ProcessJudgeResult> {
if (!llmClient.hasEndpoints()) {