Make eval pipeline skill-aware and validate SKILL manifests
This commit is contained in:
1 parent
12e731a428
commit
54bc295ead
35 files changed
+1280
-499
No files matched your search
@@ -2,8 +2,7 @@ import { z } from "zod";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import type { ToolRegistry } from "@tools/registry";
|
||||
import {
|
||||
getEvalProcessLlmJudgeMaxTokens,
|
||||
getEvalPrmEnabled,
|
||||
@@ -12,6 +11,7 @@ import {
|
||||
} from "../../config.js";
|
||||
import type {
|
||||
EvalCaseExpected,
|
||||
EvalProcessTraceSnapshot,
|
||||
ProcessJudgeCheck,
|
||||
ProcessJudgeResult,
|
||||
} from "../types.js";
|
||||
@@ -80,7 +80,7 @@ function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudge
|
||||
|
||||
function enrichWithPrm(
|
||||
result: ProcessJudgeResult,
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
): ProcessJudgeResult {
|
||||
if (!getEvalPrmEnabled()) {
|
||||
return result;
|
||||
@@ -125,7 +125,7 @@ function hasAllTools(
|
||||
return subset && actual.size === expected.size;
|
||||
}
|
||||
|
||||
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
||||
function countRecordedSteps(trace: EvalProcessTraceSnapshot): number {
|
||||
const scalarSteps = [
|
||||
trace.steps.classify,
|
||||
trace.steps.derive,
|
||||
@@ -138,18 +138,20 @@ function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
||||
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
|
||||
}
|
||||
|
||||
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
||||
function validateToolArgs(
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
toolRegistry: ToolRegistry,
|
||||
): ProcessJudgeCheck {
|
||||
const toolCalls = trace.steps.toolCalls ?? [];
|
||||
if (toolCalls.length === 0) {
|
||||
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
|
||||
}
|
||||
|
||||
const registry = createBuiltInToolRegistry();
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const call of toolCalls) {
|
||||
try {
|
||||
registry.get(call.output.name).argsSchema.parse(call.output.args);
|
||||
toolRegistry.get(call.output.name).argsSchema.parse(call.output.args);
|
||||
} catch {
|
||||
failures.push(call.output.name);
|
||||
}
|
||||
@@ -167,7 +169,8 @@ function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
||||
|
||||
export function judgeProcessRule(
|
||||
expected: EvalCaseExpected,
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
toolRegistry: ToolRegistry,
|
||||
): ProcessJudgeResult {
|
||||
const checks: ProcessJudgeCheck[] = [];
|
||||
|
||||
@@ -244,7 +247,7 @@ export function judgeProcessRule(
|
||||
);
|
||||
}
|
||||
|
||||
checks.push(validateToolArgs(trace));
|
||||
checks.push(validateToolArgs(trace, toolRegistry));
|
||||
|
||||
const toolPlanStepCount = trace.steps.plan?.output.toolPlanStepCount;
|
||||
if (toolPlanStepCount !== undefined) {
|
||||
@@ -318,7 +321,7 @@ export function judgeProcessRule(
|
||||
}
|
||||
|
||||
function buildProcessJudgePrompt(
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): string {
|
||||
return [
|
||||
@@ -341,7 +344,7 @@ function buildProcessJudgePrompt(
|
||||
|
||||
export async function judgeProcessWithLlm(
|
||||
llmClient: LlmClient,
|
||||
trace: ProcessTraceSnapshot,
|
||||
trace: EvalProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): Promise<ProcessJudgeResult> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
|
||||
Reference in new issue
Block a user