354 lines
8.6 KiB
TypeScript
354 lines
8.6 KiB
TypeScript
import { z } from "zod";
|
|
|
|
import type { AgentRequest } from "@agent/types";
|
|
import type { LlmClient } from "@llm/llm-client";
|
|
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
|
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
|
import {
|
|
getEvalProcessLlmJudgeMaxTokens,
|
|
getLlmJudgeModel,
|
|
} from "../../config.js";
|
|
import type {
|
|
EvalCaseExpected,
|
|
ProcessJudgeCheck,
|
|
ProcessJudgeResult,
|
|
} from "../types.js";
|
|
|
|
const processLlmJudgeSchema = z.object({
|
|
rationale: z.string().trim().min(1).max(300),
|
|
score: z.number().min(0).max(1),
|
|
});
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(3));
|
|
}
|
|
|
|
function makeCheck(
|
|
step: string,
|
|
rule: ProcessJudgeCheck["rule"],
|
|
passed: boolean,
|
|
detail?: string,
|
|
): ProcessJudgeCheck {
|
|
return {
|
|
step,
|
|
rule,
|
|
passed,
|
|
skipped: false,
|
|
detail,
|
|
};
|
|
}
|
|
|
|
function makeSkippedCheck(
|
|
step: string,
|
|
rule: ProcessJudgeCheck["rule"],
|
|
detail: string,
|
|
): ProcessJudgeCheck {
|
|
return {
|
|
step,
|
|
rule,
|
|
passed: true,
|
|
skipped: true,
|
|
detail,
|
|
};
|
|
}
|
|
|
|
function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudgeResult {
|
|
const activeChecks = checks.filter((check) => !check.skipped);
|
|
const passedCount = activeChecks.filter((check) => check.passed).length;
|
|
const totalCount = activeChecks.length;
|
|
const result: ProcessJudgeResult = {
|
|
checks,
|
|
passedCount,
|
|
score: totalCount === 0 ? 1 : round(passedCount / totalCount),
|
|
totalCount,
|
|
};
|
|
|
|
if (rationale) {
|
|
result.rationale = rationale;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function hasAllTools(
|
|
actualTools: string[],
|
|
expectedTools: string[],
|
|
mode: "exact" | "subset",
|
|
): boolean {
|
|
const actual = new Set(actualTools);
|
|
const expected = new Set(expectedTools);
|
|
const subset = expectedTools.every((tool) => actual.has(tool));
|
|
|
|
if (mode === "subset") {
|
|
return subset;
|
|
}
|
|
|
|
return subset && actual.size === expected.size;
|
|
}
|
|
|
|
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
|
const scalarSteps = [
|
|
trace.steps.classify,
|
|
trace.steps.derive,
|
|
trace.steps.plan,
|
|
trace.steps.respond,
|
|
trace.steps.critic,
|
|
trace.steps.validate,
|
|
].filter(Boolean).length;
|
|
|
|
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
|
|
}
|
|
|
|
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
|
const toolCalls = trace.steps.toolCalls ?? [];
|
|
if (toolCalls.length === 0) {
|
|
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
|
|
}
|
|
|
|
const registry = createBuiltInToolRegistry();
|
|
const failures: string[] = [];
|
|
|
|
for (const call of toolCalls) {
|
|
try {
|
|
registry.get(call.output.name).argsSchema.parse(call.output.args);
|
|
} catch {
|
|
failures.push(call.output.name);
|
|
}
|
|
}
|
|
|
|
return makeCheck(
|
|
"tool-call",
|
|
"tool-args-valid",
|
|
failures.length === 0,
|
|
failures.length === 0
|
|
? undefined
|
|
: `invalid args for: ${failures.join(", ")}`,
|
|
);
|
|
}
|
|
|
|
export function judgeProcessRule(
|
|
expected: EvalCaseExpected,
|
|
trace: ProcessTraceSnapshot,
|
|
): ProcessJudgeResult {
|
|
const checks: ProcessJudgeCheck[] = [];
|
|
|
|
if (expected.intent) {
|
|
const actualIntent = trace.steps.classify?.output.intent;
|
|
checks.push(
|
|
makeCheck(
|
|
"classify",
|
|
"intent-match",
|
|
actualIntent === expected.intent,
|
|
`expected=${expected.intent}, actual=${actualIntent ?? "missing"}`,
|
|
),
|
|
);
|
|
} else {
|
|
checks.push(makeSkippedCheck("classify", "intent-match", "no expected intent"));
|
|
}
|
|
|
|
const derived = trace.steps.derive?.output;
|
|
if (derived?.responseMode === "llm-with-tools") {
|
|
checks.push(
|
|
makeCheck(
|
|
"derive",
|
|
"tier-appropriate",
|
|
derived.tier >= 1,
|
|
`responseMode=${derived.responseMode}, tier=${derived.tier}`,
|
|
),
|
|
);
|
|
} else {
|
|
checks.push(
|
|
makeSkippedCheck(
|
|
"derive",
|
|
"tier-appropriate",
|
|
"responseMode is not llm-with-tools",
|
|
),
|
|
);
|
|
}
|
|
|
|
const processExpected = expected.processExpected;
|
|
if (processExpected?.expectedApproach) {
|
|
const actualApproach = trace.steps.plan?.output.approach;
|
|
checks.push(
|
|
makeCheck(
|
|
"plan",
|
|
"planner-approach",
|
|
actualApproach === processExpected.expectedApproach,
|
|
`expected=${processExpected.expectedApproach}, actual=${actualApproach ?? "missing"}`,
|
|
),
|
|
);
|
|
} else {
|
|
checks.push(
|
|
makeSkippedCheck("plan", "planner-approach", "no expected approach"),
|
|
);
|
|
}
|
|
|
|
if (processExpected?.expectedTools?.length) {
|
|
const actualTools =
|
|
trace.steps.toolCalls?.map((call) => call.output.name) ?? [];
|
|
const mode = processExpected.toolMode ?? "subset";
|
|
checks.push(
|
|
makeCheck(
|
|
"tool-call",
|
|
"tools-selected-correctly",
|
|
hasAllTools(actualTools, processExpected.expectedTools, mode),
|
|
`expected=${processExpected.expectedTools.join(",")}, actual=${actualTools.join(",")}, mode=${mode}`,
|
|
),
|
|
);
|
|
} else {
|
|
checks.push(
|
|
makeSkippedCheck(
|
|
"tool-call",
|
|
"tools-selected-correctly",
|
|
"no expected tools",
|
|
),
|
|
);
|
|
}
|
|
|
|
checks.push(validateToolArgs(trace));
|
|
|
|
if (processExpected?.minSteps !== undefined) {
|
|
const actualSteps = countRecordedSteps(trace);
|
|
checks.push(
|
|
makeCheck(
|
|
"trace",
|
|
"min-steps",
|
|
actualSteps >= processExpected.minSteps,
|
|
`expected>=${processExpected.minSteps}, actual=${actualSteps}`,
|
|
),
|
|
);
|
|
} else {
|
|
checks.push(makeSkippedCheck("trace", "min-steps", "no minSteps"));
|
|
}
|
|
|
|
const critic = trace.steps.critic?.output;
|
|
if (critic && derived?.tier === 2) {
|
|
const validationStatus = trace.steps.validate?.output.status;
|
|
checks.push(
|
|
makeCheck(
|
|
"critic",
|
|
"critic-decision-justified",
|
|
critic.decision !== "keep" || validationStatus !== "reject",
|
|
`decision=${critic.decision}, validate=${validationStatus ?? "missing"}`,
|
|
),
|
|
);
|
|
} else {
|
|
checks.push(
|
|
makeSkippedCheck(
|
|
"critic",
|
|
"critic-decision-justified",
|
|
"no tier 2 critic decision",
|
|
),
|
|
);
|
|
}
|
|
|
|
const validate = trace.steps.validate?.output;
|
|
if (validate) {
|
|
checks.push(
|
|
makeCheck(
|
|
"validate",
|
|
"validate-no-reject",
|
|
validate.status !== "reject",
|
|
`status=${validate.status}`,
|
|
),
|
|
);
|
|
} else {
|
|
checks.push(
|
|
makeCheck("validate", "validate-no-reject", false, "missing validate step"),
|
|
);
|
|
}
|
|
|
|
return finalize(checks);
|
|
}
|
|
|
|
function buildProcessJudgePrompt(
|
|
trace: ProcessTraceSnapshot,
|
|
options: { finalText: string; request: AgentRequest },
|
|
): string {
|
|
return [
|
|
"온보딩 에이전트의 단계별 실행 trace를 평가해 주세요.",
|
|
"planner 선택, tool 선택/인자, critic 판단, 최종 검증 흐름이 합리적인지 보세요.",
|
|
"score는 0부터 1 사이 숫자입니다.",
|
|
"",
|
|
"[request]",
|
|
JSON.stringify(options.request),
|
|
"",
|
|
"[trace]",
|
|
JSON.stringify(trace),
|
|
"",
|
|
"[finalText]",
|
|
options.finalText,
|
|
"",
|
|
'JSON object only: {"score":0.8,"rationale":"..."}',
|
|
].join("\n");
|
|
}
|
|
|
|
export async function judgeProcessWithLlm(
|
|
llmClient: LlmClient,
|
|
trace: ProcessTraceSnapshot,
|
|
options: { finalText: string; request: AgentRequest },
|
|
): Promise<ProcessJudgeResult> {
|
|
if (!llmClient.hasEndpoints()) {
|
|
return finalize(
|
|
[
|
|
makeSkippedCheck(
|
|
"trace",
|
|
"validate-no-reject",
|
|
"no LLM endpoint for process judge",
|
|
),
|
|
],
|
|
"skipped: no LLM endpoint",
|
|
);
|
|
}
|
|
|
|
const response = await llmClient.chatCompletion(
|
|
[
|
|
{
|
|
role: "system",
|
|
content:
|
|
"당신은 온보딩 에이전트의 과정 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
|
|
},
|
|
{
|
|
role: "user",
|
|
content: buildProcessJudgePrompt(trace, options),
|
|
},
|
|
],
|
|
{
|
|
jsonMode: true,
|
|
maxTokens: getEvalProcessLlmJudgeMaxTokens(),
|
|
modelOverride: getLlmJudgeModel(),
|
|
role: "critic",
|
|
temperature: 0,
|
|
tier: 2,
|
|
},
|
|
);
|
|
|
|
if (!response) {
|
|
return finalize(
|
|
[makeCheck("trace", "validate-no-reject", false, "judge call failed")],
|
|
"process judge call failed",
|
|
);
|
|
}
|
|
|
|
try {
|
|
const parsed = processLlmJudgeSchema.parse(JSON.parse(response.content));
|
|
const passed = parsed.score >= 0.7;
|
|
return finalize(
|
|
[
|
|
makeCheck(
|
|
"trace",
|
|
"validate-no-reject",
|
|
passed,
|
|
`llmScore=${round(parsed.score)}`,
|
|
),
|
|
],
|
|
parsed.rationale,
|
|
);
|
|
} catch {
|
|
return finalize(
|
|
[makeCheck("trace", "validate-no-reject", false, "invalid judge json")],
|
|
"invalid process judge json",
|
|
);
|
|
}
|
|
}
|