.
This commit is contained in:
1 parent
1453433b62
commit
da76af3826
65 files changed
+11089
-143
No files matched your search
@@ -0,0 +1,353 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import {
|
||||
getEvalProcessLlmJudgeMaxTokens,
|
||||
getLlmJudgeModel,
|
||||
} from "../../config.js";
|
||||
import type {
|
||||
EvalCaseExpected,
|
||||
ProcessJudgeCheck,
|
||||
ProcessJudgeResult,
|
||||
} from "../types.js";
|
||||
|
||||
const processLlmJudgeSchema = z.object({
|
||||
rationale: z.string().trim().min(1).max(300),
|
||||
score: z.number().min(0).max(1),
|
||||
});
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function makeCheck(
|
||||
step: string,
|
||||
rule: ProcessJudgeCheck["rule"],
|
||||
passed: boolean,
|
||||
detail?: string,
|
||||
): ProcessJudgeCheck {
|
||||
return {
|
||||
step,
|
||||
rule,
|
||||
passed,
|
||||
skipped: false,
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
function makeSkippedCheck(
|
||||
step: string,
|
||||
rule: ProcessJudgeCheck["rule"],
|
||||
detail: string,
|
||||
): ProcessJudgeCheck {
|
||||
return {
|
||||
step,
|
||||
rule,
|
||||
passed: true,
|
||||
skipped: true,
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
function finalize(checks: ProcessJudgeCheck[], rationale?: string): ProcessJudgeResult {
|
||||
const activeChecks = checks.filter((check) => !check.skipped);
|
||||
const passedCount = activeChecks.filter((check) => check.passed).length;
|
||||
const totalCount = activeChecks.length;
|
||||
const result: ProcessJudgeResult = {
|
||||
checks,
|
||||
passedCount,
|
||||
score: totalCount === 0 ? 1 : round(passedCount / totalCount),
|
||||
totalCount,
|
||||
};
|
||||
|
||||
if (rationale) {
|
||||
result.rationale = rationale;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function hasAllTools(
|
||||
actualTools: string[],
|
||||
expectedTools: string[],
|
||||
mode: "exact" | "subset",
|
||||
): boolean {
|
||||
const actual = new Set(actualTools);
|
||||
const expected = new Set(expectedTools);
|
||||
const subset = expectedTools.every((tool) => actual.has(tool));
|
||||
|
||||
if (mode === "subset") {
|
||||
return subset;
|
||||
}
|
||||
|
||||
return subset && actual.size === expected.size;
|
||||
}
|
||||
|
||||
function countRecordedSteps(trace: ProcessTraceSnapshot): number {
|
||||
const scalarSteps = [
|
||||
trace.steps.classify,
|
||||
trace.steps.derive,
|
||||
trace.steps.plan,
|
||||
trace.steps.respond,
|
||||
trace.steps.critic,
|
||||
trace.steps.validate,
|
||||
].filter(Boolean).length;
|
||||
|
||||
return scalarSteps + (trace.steps.toolCalls?.length ?? 0);
|
||||
}
|
||||
|
||||
function validateToolArgs(trace: ProcessTraceSnapshot): ProcessJudgeCheck {
|
||||
const toolCalls = trace.steps.toolCalls ?? [];
|
||||
if (toolCalls.length === 0) {
|
||||
return makeSkippedCheck("tool-call", "tool-args-valid", "no tool calls");
|
||||
}
|
||||
|
||||
const registry = createBuiltInToolRegistry();
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const call of toolCalls) {
|
||||
try {
|
||||
registry.get(call.output.name).argsSchema.parse(call.output.args);
|
||||
} catch {
|
||||
failures.push(call.output.name);
|
||||
}
|
||||
}
|
||||
|
||||
return makeCheck(
|
||||
"tool-call",
|
||||
"tool-args-valid",
|
||||
failures.length === 0,
|
||||
failures.length === 0
|
||||
? undefined
|
||||
: `invalid args for: ${failures.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function judgeProcessRule(
|
||||
expected: EvalCaseExpected,
|
||||
trace: ProcessTraceSnapshot,
|
||||
): ProcessJudgeResult {
|
||||
const checks: ProcessJudgeCheck[] = [];
|
||||
|
||||
if (expected.intent) {
|
||||
const actualIntent = trace.steps.classify?.output.intent;
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"classify",
|
||||
"intent-match",
|
||||
actualIntent === expected.intent,
|
||||
`expected=${expected.intent}, actual=${actualIntent ?? "missing"}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(makeSkippedCheck("classify", "intent-match", "no expected intent"));
|
||||
}
|
||||
|
||||
const derived = trace.steps.derive?.output;
|
||||
if (derived?.responseMode === "llm-with-tools") {
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"derive",
|
||||
"tier-appropriate",
|
||||
derived.tier >= 1,
|
||||
`responseMode=${derived.responseMode}, tier=${derived.tier}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck(
|
||||
"derive",
|
||||
"tier-appropriate",
|
||||
"responseMode is not llm-with-tools",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const processExpected = expected.processExpected;
|
||||
if (processExpected?.expectedApproach) {
|
||||
const actualApproach = trace.steps.plan?.output.approach;
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"plan",
|
||||
"planner-approach",
|
||||
actualApproach === processExpected.expectedApproach,
|
||||
`expected=${processExpected.expectedApproach}, actual=${actualApproach ?? "missing"}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck("plan", "planner-approach", "no expected approach"),
|
||||
);
|
||||
}
|
||||
|
||||
if (processExpected?.expectedTools?.length) {
|
||||
const actualTools =
|
||||
trace.steps.toolCalls?.map((call) => call.output.name) ?? [];
|
||||
const mode = processExpected.toolMode ?? "subset";
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"tool-call",
|
||||
"tools-selected-correctly",
|
||||
hasAllTools(actualTools, processExpected.expectedTools, mode),
|
||||
`expected=${processExpected.expectedTools.join(",")}, actual=${actualTools.join(",")}, mode=${mode}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck(
|
||||
"tool-call",
|
||||
"tools-selected-correctly",
|
||||
"no expected tools",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
checks.push(validateToolArgs(trace));
|
||||
|
||||
if (processExpected?.minSteps !== undefined) {
|
||||
const actualSteps = countRecordedSteps(trace);
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"trace",
|
||||
"min-steps",
|
||||
actualSteps >= processExpected.minSteps,
|
||||
`expected>=${processExpected.minSteps}, actual=${actualSteps}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(makeSkippedCheck("trace", "min-steps", "no minSteps"));
|
||||
}
|
||||
|
||||
const critic = trace.steps.critic?.output;
|
||||
if (critic && derived?.tier === 2) {
|
||||
const validationStatus = trace.steps.validate?.output.status;
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"critic",
|
||||
"critic-decision-justified",
|
||||
critic.decision !== "keep" || validationStatus !== "reject",
|
||||
`decision=${critic.decision}, validate=${validationStatus ?? "missing"}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeSkippedCheck(
|
||||
"critic",
|
||||
"critic-decision-justified",
|
||||
"no tier 2 critic decision",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const validate = trace.steps.validate?.output;
|
||||
if (validate) {
|
||||
checks.push(
|
||||
makeCheck(
|
||||
"validate",
|
||||
"validate-no-reject",
|
||||
validate.status !== "reject",
|
||||
`status=${validate.status}`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
makeCheck("validate", "validate-no-reject", false, "missing validate step"),
|
||||
);
|
||||
}
|
||||
|
||||
return finalize(checks);
|
||||
}
|
||||
|
||||
function buildProcessJudgePrompt(
|
||||
trace: ProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): string {
|
||||
return [
|
||||
"온보딩 에이전트의 단계별 실행 trace를 평가해 주세요.",
|
||||
"planner 선택, tool 선택/인자, critic 판단, 최종 검증 흐름이 합리적인지 보세요.",
|
||||
"score는 0부터 1 사이 숫자입니다.",
|
||||
"",
|
||||
"[request]",
|
||||
JSON.stringify(options.request),
|
||||
"",
|
||||
"[trace]",
|
||||
JSON.stringify(trace),
|
||||
"",
|
||||
"[finalText]",
|
||||
options.finalText,
|
||||
"",
|
||||
'JSON object only: {"score":0.8,"rationale":"..."}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function judgeProcessWithLlm(
|
||||
llmClient: LlmClient,
|
||||
trace: ProcessTraceSnapshot,
|
||||
options: { finalText: string; request: AgentRequest },
|
||||
): Promise<ProcessJudgeResult> {
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
return finalize(
|
||||
[
|
||||
makeSkippedCheck(
|
||||
"trace",
|
||||
"validate-no-reject",
|
||||
"no LLM endpoint for process judge",
|
||||
),
|
||||
],
|
||||
"skipped: no LLM endpoint",
|
||||
);
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 에이전트의 과정 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildProcessJudgePrompt(trace, options),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: getEvalProcessLlmJudgeMaxTokens(),
|
||||
modelOverride: getLlmJudgeModel(),
|
||||
role: "critic",
|
||||
temperature: 0,
|
||||
tier: 2,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
return finalize(
|
||||
[makeCheck("trace", "validate-no-reject", false, "judge call failed")],
|
||||
"process judge call failed",
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = processLlmJudgeSchema.parse(JSON.parse(response.content));
|
||||
const passed = parsed.score >= 0.7;
|
||||
return finalize(
|
||||
[
|
||||
makeCheck(
|
||||
"trace",
|
||||
"validate-no-reject",
|
||||
passed,
|
||||
`llmScore=${round(parsed.score)}`,
|
||||
),
|
||||
],
|
||||
parsed.rationale,
|
||||
);
|
||||
} catch {
|
||||
return finalize(
|
||||
[makeCheck("trace", "validate-no-reject", false, "invalid judge json")],
|
||||
"invalid process judge json",
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user