import { z } from "zod"; import type { AgentRequest } from "@agent/types"; import type { LlmClient } from "@llm/llm-client"; import { createBuiltInToolRegistry } from "@tools/builtins/index"; import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import { getEvalProcessLlmJudgeMaxTokens, getLlmJudgeModel, } from "../../config.js"; import type { EvalCaseExpected, ProcessJudgeCheck, ProcessJudgeResult, } from "../types.js"; const processLlmJudgeSchema = z.object({ rationale: z.string().trim().min(1).max(300), score: z.number().min(0).max(1), }); function round(value: number): number { return Number(value.toFixed(3)); } function makeCheck( step: string, rule: ProcessJudgeCheck["rule"], passed: boolean, detail?: string, ): ProcessJudgeCheck { return { step, rule, passed, skipped: false, detail, }; } function makeSkippedCheck( step: string, rule: ProcessJudgeCheck["rule"], detail: string, ): ProcessJudgeCheck { return { step, rule, passed: true, skipped: true, detail, }; } function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudgeResult { const activeChecks = checks.filter((check) => !check.skipped); const passedCount = activeChecks.filter((check) => check.passed).length; const totalCount = activeChecks.length; const result: ProcessJudgeResult = { checks, passedCount, score: totalCount === 0 ? 1 : round(passedCount / totalCount), totalCount, }; if (rationale) { result.rationale = rationale; } return result; } function hasAllTools( actualTools: string[], expectedTools: string[], mode: "exact" | "subset", ): boolean { const actual = new Set(actualTools); const expected = new Set(expectedTools); const subset = expectedTools.every((tool) => actual.has(tool)); if (mode === "subset") { return subset; } return subset && actual.size === expected.size; } function countRecordedSteps(trace: ProcessTraceSnapshot): number { const scalarSteps = [ trace.steps.classify, trace.steps.derive, trace.steps.plan, trace.steps.respond, trace.steps.critic, trace.steps.validate, ].filter(Boolean).length; return scalarSteps + (trace.steps.toolCalls?.length ?? 0); } function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck { const toolCalls = trace.steps.toolCalls ?? []; if (toolCalls.length === 0) { return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls"); } const registry = createBuiltInToolRegistry(); const failures: string[] = []; for (const call of toolCalls) { try { registry.get(call.output.name).argsSchema.parse(call.output.args); } catch { failures.push(call.output.name); } } return makeCheck( "tool-call", "tool-args-valid", failures.length === 0, failures.length === 0 ? undefined : `invalid args for: ${failures.join(", ")}`, ); } export function judgeProcessRule( expected: EvalCaseExpected, trace: ProcessTraceSnapshot, ): ProcessJudgeResult { const checks: ProcessJudgeCheck[] = []; if (expected.intent) { const actualIntent = trace.steps.classify?.output.intent; checks.push( makeCheck( "classify", "intent-match", actualIntent === expected.intent, `expected=${expected.intent}, actual=${actualIntent ?? "missing"}`, ), ); } else { checks.push(makeSkippedCheck("classify", "intent-match", "no expected intent")); } const derived = trace.steps.derive?.output; if (derived?.responseMode === "llm-with-tools") { checks.push( makeCheck( "derive", "tier-appropriate", derived.tier >= 1, `responseMode=${derived.responseMode}, tier=${derived.tier}`, ), ); } else { checks.push( makeSkippedCheck( "derive", "tier-appropriate", "responseMode is not llm-with-tools", ), ); } const processExpected = expected.processExpected; if (processExpected?.expectedApproach) { const actualApproach = trace.steps.plan?.output.approach; checks.push( makeCheck( "plan", "planner-approach", actualApproach === processExpected.expectedApproach, `expected=${processExpected.expectedApproach}, actual=${actualApproach ?? "missing"}`, ), ); } else { checks.push( makeSkippedCheck("plan", "planner-approach", "no expected approach"), ); } if (processExpected?.expectedTools?.length) { const actualTools = trace.steps.toolCalls?.map((call) => call.output.name) ?? []; const mode = processExpected.toolMode ?? "subset"; checks.push( makeCheck( "tool-call", "tools-selected-correctly", hasAllTools(actualTools, processExpected.expectedTools, mode), `expected=${processExpected.expectedTools.join(",")}, actual=${actualTools.join(",")}, mode=${mode}`, ), ); } else { checks.push( makeSkippedCheck( "tool-call", "tools-selected-correctly", "no expected tools", ), ); } checks.push(validateToolArgs(trace)); if (processExpected?.minSteps !== undefined) { const actualSteps = countRecordedSteps(trace); checks.push( makeCheck( "trace", "min-steps", actualSteps >= processExpected.minSteps, `expected>=${processExpected.minSteps}, actual=${actualSteps}`, ), ); } else { checks.push(makeSkippedCheck("trace", "min-steps", "no minSteps")); } const critic = trace.steps.critic?.output; if (critic && derived?.tier === 2) { const validationStatus = trace.steps.validate?.output.status; checks.push( makeCheck( "critic", "critic-decision-justified", critic.decision !== "keep" || validationStatus !== "reject", `decision=${critic.decision}, validate=${validationStatus ?? "missing"}`, ), ); } else { checks.push( makeSkippedCheck( "critic", "critic-decision-justified", "no tier 2 critic decision", ), ); } const validate = trace.steps.validate?.output; if (validate) { checks.push( makeCheck( "validate", "validate-no-reject", validate.status !== "reject", `status=${validate.status}`, ), ); } else { checks.push( makeCheck("validate", "validate-no-reject", false, "missing validate step"), ); } return finalize(checks); } function buildProcessJudgePrompt( trace: ProcessTraceSnapshot, options: { finalText: string; request: AgentRequest }, ): string { return [ "온보딩 에이전트의 단계별 실행 trace를 평가해 주세요.", "planner 선택, tool 선택/인자, critic 판단, 최종 검증 흐름이 합리적인지 보세요.", "score는 0부터 1 사이 숫자입니다.", "", "[request]", JSON.stringify(options.request), "", "[trace]", JSON.stringify(trace), "", "[finalText]", options.finalText, "", 'JSON object only: {"score":0.8,"rationale":"..."}', ].join("\n"); } export async function judgeProcessWithLlm( llmClient: LlmClient, trace: ProcessTraceSnapshot, options: { finalText: string; request: AgentRequest }, ): Promise { if (!llmClient.hasEndpoints()) { return finalize( [ makeSkippedCheck( "trace", "validate-no-reject", "no LLM endpoint for process judge", ), ], "skipped: no LLM endpoint", ); } const response = await llmClient.chatCompletion( [ { role: "system", content: "당신은 온보딩 에이전트의 과정 품질 평가자입니다. 반드시 JSON object만 출력하세요.", }, { role: "user", content: buildProcessJudgePrompt(trace, options), }, ], { jsonMode: true, maxTokens: getEvalProcessLlmJudgeMaxTokens(), modelOverride: getLlmJudgeModel(), role: "critic", temperature: 0, tier: 2, }, ); if (!response) { return finalize( [makeCheck("trace", "validate-no-reject", false, "judge call failed")], "process judge call failed", ); } try { const parsed = processLlmJudgeSchema.parse(JSON.parse(response.content)); const passed = parsed.score >= 0.7; return finalize( [ makeCheck( "trace", "validate-no-reject", passed, `llmScore=${round(parsed.score)}`, ), ], parsed.rationale, ); } catch { return finalize( [makeCheck("trace", "validate-no-reject", false, "invalid judge json")], "invalid process judge json", ); } }