import { mkdtempSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { afterEach, describe, expect, it } from "vitest"; import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import { judgeProcessRule } from "./process-judge.js"; import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js"; function makeTrace( overrides?: Partial, ): ProcessTraceSnapshot { return { steps: { classify: { name: "classify", startedAtMs: 1, durationMs: 1, output: { intent: "status-question", confidence: 0.9, source: "regex", }, }, derive: { name: "derive", startedAtMs: 2, durationMs: 1, output: { phase: "photo-pending", responseMode: "llm-with-tools", shouldRespond: true, tier: 1, }, }, plan: { name: "plan", startedAtMs: 3, durationMs: 1, output: { approach: "validate-then-explain", mustAvoid: [], mustInclude: ["남은 절차"], suggestedTools: ["validate-nickname"], targetLengthChars: 120, }, }, toolCalls: [ { name: "tool-call", startedAtMs: 4, durationMs: 1, output: { name: "validate-nickname", args: { nickname: "프리리/95/여/D", }, ok: true, resultSummary: '{"valid":true}', }, }, ], respond: { name: "respond", startedAtMs: 5, durationMs: 1, output: { draftText: "방 하트 인증만 남았어요.", tokensIn: 0, tokensOut: 0, }, }, validate: { name: "validate", startedAtMs: 6, durationMs: 1, output: { status: "pass", reasons: [], }, }, ...overrides, }, }; } describe("judgeProcessRule", () => { afterEach(() => { delete process.env["EVAL_PRM_ENABLED"]; delete process.env["EVAL_PRM_WEIGHTS_PATH"]; }); it("passes active process expectations", () => { const result = judgeProcessRule( { intent: "status-question", processExpected: { expectedApproach: "validate-then-explain", expectedTools: ["validate-nickname"], minSteps: 5, }, }, makeTrace(), ); expect(result.score).toBe(1); expect(result.totalCount).toBeGreaterThan(0); expect(result.checks.every((check) => check.skipped || check.passed)).toBe( true, ); }); it("fails intent, tool, and approach mismatches", () => { const result = judgeProcessRule( { intent: "photo-question", processExpected: { expectedApproach: "answer-direct", expectedTools: ["check-onboarding-progress"], toolMode: "exact", }, }, makeTrace(), ); expect( result.checks.find((check) => check.rule === "intent-match")?.passed, ).toBe(false); expect( result.checks.find((check) => check.rule === "planner-approach") ?.passed, ).toBe(false); expect( result.checks.find((check) => check.rule === "tools-selected-correctly") ?.passed, ).toBe(false); }); it("fails invalid tool arguments", () => { const result = judgeProcessRule( {}, makeTrace({ toolCalls: [ { name: "tool-call", startedAtMs: 4, durationMs: 1, output: { name: "validate-nickname", args: {}, ok: false, resultSummary: "invalid", }, }, ], }), ); expect( result.checks.find((check) => check.rule === "tool-args-valid")?.passed, ).toBe(false); }); it("checks toolPlan step count against recorded tool calls", () => { const result = judgeProcessRule( {}, makeTrace({ plan: { name: "plan", startedAtMs: 3, durationMs: 1, output: { approach: "validate-then-explain", mustAvoid: [], mustInclude: ["남은 절차"], suggestedTools: ["validate-nickname"], targetLengthChars: 120, toolPlanStepCount: 2, }, }, }), ); expect( result.checks.find((check) => check.rule === "tool-plan-consistent") ?.passed, ).toBe(false); }); it("fails tier and validator process regressions", () => { const result = judgeProcessRule( {}, makeTrace({ derive: { name: "derive", startedAtMs: 2, durationMs: 1, output: { phase: "photo-pending", responseMode: "llm-with-tools", shouldRespond: true, tier: 0, }, }, validate: { name: "validate", startedAtMs: 6, durationMs: 1, output: { status: "reject", reasons: ["stateConsistency"], }, }, }), ); expect( result.checks.find((check) => check.rule === "tier-appropriate")?.passed, ).toBe(false); expect( result.checks.find((check) => check.rule === "validate-no-reject") ?.passed, ).toBe(false); }); it("skips optional expectations when a golden case has not opted in", () => { const result = judgeProcessRule({}, makeTrace()); expect( result.checks.find((check) => check.rule === "intent-match")?.skipped, ).toBe(true); expect( result.checks.find((check) => check.rule === "planner-approach") ?.skipped, ).toBe(true); }); it("adds PRM features, score, and attribution when enabled", () => { const weightsPath = join(mkdtempSync(join(tmpdir(), "prm-process-")), "weights.json"); savePrmModel(weightsPath, { featureNames: [...PRM_FEATURE_NAMES], trainedAt: "2026-05-01T00:00:00.000Z", version: 1, weights: [1, 0, 0, 0, 0, 0, 0, 4, 0, 0], }); process.env["EVAL_PRM_ENABLED"] = "true"; process.env["EVAL_PRM_WEIGHTS_PATH"] = weightsPath; const result = judgeProcessRule( {}, makeTrace({ validate: { name: "validate", startedAtMs: 6, durationMs: 1, output: { status: "reject", reasons: ["stateConsistency"], }, }, }), ); expect(result.prmFeatures).toMatchObject({ validateStatus: 0, }); expect(result.prmScore).toBeDefined(); expect(result.rootCauseStep).toBe("validate"); }); });