import { describe, expect, it } from "vitest"; import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import { judgeProcessRule } from "./process-judge.js"; function makeTrace( overrides?: Partial, ): ProcessTraceSnapshot { return { steps: { classify: { name: "classify", startedAtMs: 1, durationMs: 1, output: { intent: "status-question", confidence: 0.9, source: "regex", }, }, derive: { name: "derive", startedAtMs: 2, durationMs: 1, output: { phase: "photo-pending", responseMode: "llm-with-tools", shouldRespond: true, tier: 1, }, }, plan: { name: "plan", startedAtMs: 3, durationMs: 1, output: { approach: "validate-then-explain", mustAvoid: [], mustInclude: ["남은 절차"], suggestedTools: ["validate-nickname"], targetLengthChars: 120, }, }, toolCalls: [ { name: "tool-call", startedAtMs: 4, durationMs: 1, output: { name: "validate-nickname", args: { nickname: "프리리/95/여/D", }, ok: true, resultSummary: '{"valid":true}', }, }, ], respond: { name: "respond", startedAtMs: 5, durationMs: 1, output: { draftText: "방 하트 인증만 남았어요.", tokensIn: 0, tokensOut: 0, }, }, validate: { name: "validate", startedAtMs: 6, durationMs: 1, output: { status: "pass", reasons: [], }, }, ...overrides, }, }; } describe("judgeProcessRule", () => { it("passes active process expectations", () => { const result = judgeProcessRule( { intent: "status-question", processExpected: { expectedApproach: "validate-then-explain", expectedTools: ["validate-nickname"], minSteps: 5, }, }, makeTrace(), ); expect(result.score).toBe(1); expect(result.totalCount).toBeGreaterThan(0); expect(result.checks.every((check) => check.skipped || check.passed)).toBe( true, ); }); it("fails intent, tool, and approach mismatches", () => { const result = judgeProcessRule( { intent: "photo-question", processExpected: { expectedApproach: "answer-direct", expectedTools: ["check-onboarding-progress"], toolMode: "exact", }, }, makeTrace(), ); expect( result.checks.find((check) => check.rule === "intent-match")?.passed, ).toBe(false); expect( result.checks.find((check) => check.rule === "planner-approach") ?.passed, ).toBe(false); expect( result.checks.find((check) => check.rule === "tools-selected-correctly") ?.passed, ).toBe(false); }); it("fails invalid tool arguments", () => { const result = judgeProcessRule( {}, makeTrace({ toolCalls: [ { name: "tool-call", startedAtMs: 4, durationMs: 1, output: { name: "validate-nickname", args: {}, ok: false, resultSummary: "invalid", }, }, ], }), ); expect( result.checks.find((check) => check.rule === "tool-args-valid")?.passed, ).toBe(false); }); it("fails tier and validator process regressions", () => { const result = judgeProcessRule( {}, makeTrace({ derive: { name: "derive", startedAtMs: 2, durationMs: 1, output: { phase: "photo-pending", responseMode: "llm-with-tools", shouldRespond: true, tier: 0, }, }, validate: { name: "validate", startedAtMs: 6, durationMs: 1, output: { status: "reject", reasons: ["stateConsistency"], }, }, }), ); expect( result.checks.find((check) => check.rule === "tier-appropriate")?.passed, ).toBe(false); expect( result.checks.find((check) => check.rule === "validate-no-reject") ?.passed, ).toBe(false); }); it("skips optional expectations when a golden case has not opted in", () => { const result = judgeProcessRule({}, makeTrace()); expect( result.checks.find((check) => check.rule === "intent-match")?.skipped, ).toBe(true); expect( result.checks.find((check) => check.rule === "planner-approach") ?.skipped, ).toBe(true); }); });