203 lines
4.8 KiB
TypeScript
203 lines
4.8 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
|
import { judgeProcessRule } from "./process-judge.js";
|
|
|
|
function makeTrace(
|
|
overrides?: Partial<ProcessTraceSnapshot["steps"]>,
|
|
): ProcessTraceSnapshot {
|
|
return {
|
|
steps: {
|
|
classify: {
|
|
name: "classify",
|
|
startedAtMs: 1,
|
|
durationMs: 1,
|
|
output: {
|
|
intent: "status-question",
|
|
confidence: 0.9,
|
|
source: "regex",
|
|
},
|
|
},
|
|
derive: {
|
|
name: "derive",
|
|
startedAtMs: 2,
|
|
durationMs: 1,
|
|
output: {
|
|
phase: "photo-pending",
|
|
responseMode: "llm-with-tools",
|
|
shouldRespond: true,
|
|
tier: 1,
|
|
},
|
|
},
|
|
plan: {
|
|
name: "plan",
|
|
startedAtMs: 3,
|
|
durationMs: 1,
|
|
output: {
|
|
approach: "validate-then-explain",
|
|
mustAvoid: [],
|
|
mustInclude: ["남은 절차"],
|
|
suggestedTools: ["validate-nickname"],
|
|
targetLengthChars: 120,
|
|
},
|
|
},
|
|
toolCalls: [
|
|
{
|
|
name: "tool-call",
|
|
startedAtMs: 4,
|
|
durationMs: 1,
|
|
output: {
|
|
name: "validate-nickname",
|
|
args: {
|
|
nickname: "프리리/95/여/D",
|
|
},
|
|
ok: true,
|
|
resultSummary: '{"valid":true}',
|
|
},
|
|
},
|
|
],
|
|
respond: {
|
|
name: "respond",
|
|
startedAtMs: 5,
|
|
durationMs: 1,
|
|
output: {
|
|
draftText: "방 하트 인증만 남았어요.",
|
|
tokensIn: 0,
|
|
tokensOut: 0,
|
|
},
|
|
},
|
|
validate: {
|
|
name: "validate",
|
|
startedAtMs: 6,
|
|
durationMs: 1,
|
|
output: {
|
|
status: "pass",
|
|
reasons: [],
|
|
},
|
|
},
|
|
...overrides,
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("judgeProcessRule", () => {
|
|
it("passes active process expectations", () => {
|
|
const result = judgeProcessRule(
|
|
{
|
|
intent: "status-question",
|
|
processExpected: {
|
|
expectedApproach: "validate-then-explain",
|
|
expectedTools: ["validate-nickname"],
|
|
minSteps: 5,
|
|
},
|
|
},
|
|
makeTrace(),
|
|
);
|
|
|
|
expect(result.score).toBe(1);
|
|
expect(result.totalCount).toBeGreaterThan(0);
|
|
expect(result.checks.every((check) => check.skipped || check.passed)).toBe(
|
|
true,
|
|
);
|
|
});
|
|
|
|
it("fails intent, tool, and approach mismatches", () => {
|
|
const result = judgeProcessRule(
|
|
{
|
|
intent: "photo-question",
|
|
processExpected: {
|
|
expectedApproach: "answer-direct",
|
|
expectedTools: ["check-onboarding-progress"],
|
|
toolMode: "exact",
|
|
},
|
|
},
|
|
makeTrace(),
|
|
);
|
|
|
|
expect(
|
|
result.checks.find((check) => check.rule === "intent-match")?.passed,
|
|
).toBe(false);
|
|
expect(
|
|
result.checks.find((check) => check.rule === "planner-approach")
|
|
?.passed,
|
|
).toBe(false);
|
|
expect(
|
|
result.checks.find((check) => check.rule === "tools-selected-correctly")
|
|
?.passed,
|
|
).toBe(false);
|
|
});
|
|
|
|
it("fails invalid tool arguments", () => {
|
|
const result = judgeProcessRule(
|
|
{},
|
|
makeTrace({
|
|
toolCalls: [
|
|
{
|
|
name: "tool-call",
|
|
startedAtMs: 4,
|
|
durationMs: 1,
|
|
output: {
|
|
name: "validate-nickname",
|
|
args: {},
|
|
ok: false,
|
|
resultSummary: "invalid",
|
|
},
|
|
},
|
|
],
|
|
}),
|
|
);
|
|
|
|
expect(
|
|
result.checks.find((check) => check.rule === "tool-args-valid")?.passed,
|
|
).toBe(false);
|
|
});
|
|
|
|
it("fails tier and validator process regressions", () => {
|
|
const result = judgeProcessRule(
|
|
{},
|
|
makeTrace({
|
|
derive: {
|
|
name: "derive",
|
|
startedAtMs: 2,
|
|
durationMs: 1,
|
|
output: {
|
|
phase: "photo-pending",
|
|
responseMode: "llm-with-tools",
|
|
shouldRespond: true,
|
|
tier: 0,
|
|
},
|
|
},
|
|
validate: {
|
|
name: "validate",
|
|
startedAtMs: 6,
|
|
durationMs: 1,
|
|
output: {
|
|
status: "reject",
|
|
reasons: ["stateConsistency"],
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
|
|
expect(
|
|
result.checks.find((check) => check.rule === "tier-appropriate")?.passed,
|
|
).toBe(false);
|
|
expect(
|
|
result.checks.find((check) => check.rule === "validate-no-reject")
|
|
?.passed,
|
|
).toBe(false);
|
|
});
|
|
|
|
it("skips optional expectations when a golden case has not opted in", () => {
|
|
const result = judgeProcessRule({}, makeTrace());
|
|
|
|
expect(
|
|
result.checks.find((check) => check.rule === "intent-match")?.skipped,
|
|
).toBe(true);
|
|
expect(
|
|
result.checks.find((check) => check.rule === "planner-approach")
|
|
?.skipped,
|
|
).toBe(true);
|
|
});
|
|
});
|