This commit is contained in:
dahyeon committed 2026-05-12 08:16:31 -07:00
1 parent 1453433b62
commit da76af3826
65 files changed
+11089 -143

No files matched your search

+202
View File
@@ -0,0 +1,202 @@
import { describe, expect, it } from "vitest";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import { judgeProcessRule } from "./process-judge.js";
function makeTrace(
overrides?: Partial<ProcessTraceSnapshot["steps"]>,
): ProcessTraceSnapshot {
return {
steps: {
classify: {
name: "classify",
startedAtMs: 1,
durationMs: 1,
output: {
intent: "status-question",
confidence: 0.9,
source: "regex",
},
},
derive: {
name: "derive",
startedAtMs: 2,
durationMs: 1,
output: {
phase: "photo-pending",
responseMode: "llm-with-tools",
shouldRespond: true,
tier: 1,
},
},
plan: {
name: "plan",
startedAtMs: 3,
durationMs: 1,
output: {
approach: "validate-then-explain",
mustAvoid: [],
mustInclude: ["남은 절차"],
suggestedTools: ["validate-nickname"],
targetLengthChars: 120,
},
},
toolCalls: [
{
name: "tool-call",
startedAtMs: 4,
durationMs: 1,
output: {
name: "validate-nickname",
args: {
nickname: "프리리/95/여/D",
},
ok: true,
resultSummary: '{"valid":true}',
},
},
],
respond: {
name: "respond",
startedAtMs: 5,
durationMs: 1,
output: {
draftText: "방 하트 인증만 남았어요.",
tokensIn: 0,
tokensOut: 0,
},
},
validate: {
name: "validate",
startedAtMs: 6,
durationMs: 1,
output: {
status: "pass",
reasons: [],
},
},
...overrides,
},
};
}
describe("judgeProcessRule", () => {
it("passes active process expectations", () => {
const result = judgeProcessRule(
{
intent: "status-question",
processExpected: {
expectedApproach: "validate-then-explain",
expectedTools: ["validate-nickname"],
minSteps: 5,
},
},
makeTrace(),
);
expect(result.score).toBe(1);
expect(result.totalCount).toBeGreaterThan(0);
expect(result.checks.every((check) => check.skipped || check.passed)).toBe(
true,
);
});
it("fails intent, tool, and approach mismatches", () => {
const result = judgeProcessRule(
{
intent: "photo-question",
processExpected: {
expectedApproach: "answer-direct",
expectedTools: ["check-onboarding-progress"],
toolMode: "exact",
},
},
makeTrace(),
);
expect(
result.checks.find((check) => check.rule === "intent-match")?.passed,
).toBe(false);
expect(
result.checks.find((check) => check.rule === "planner-approach")
?.passed,
).toBe(false);
expect(
result.checks.find((check) => check.rule === "tools-selected-correctly")
?.passed,
).toBe(false);
});
it("fails invalid tool arguments", () => {
const result = judgeProcessRule(
{},
makeTrace({
toolCalls: [
{
name: "tool-call",
startedAtMs: 4,
durationMs: 1,
output: {
name: "validate-nickname",
args: {},
ok: false,
resultSummary: "invalid",
},
},
],
}),
);
expect(
result.checks.find((check) => check.rule === "tool-args-valid")?.passed,
).toBe(false);
});
it("fails tier and validator process regressions", () => {
const result = judgeProcessRule(
{},
makeTrace({
derive: {
name: "derive",
startedAtMs: 2,
durationMs: 1,
output: {
phase: "photo-pending",
responseMode: "llm-with-tools",
shouldRespond: true,
tier: 0,
},
},
validate: {
name: "validate",
startedAtMs: 6,
durationMs: 1,
output: {
status: "reject",
reasons: ["stateConsistency"],
},
},
}),
);
expect(
result.checks.find((check) => check.rule === "tier-appropriate")?.passed,
).toBe(false);
expect(
result.checks.find((check) => check.rule === "validate-no-reject")
?.passed,
).toBe(false);
});
it("skips optional expectations when a golden case has not opted in", () => {
const result = judgeProcessRule({}, makeTrace());
expect(
result.checks.find((check) => check.rule === "intent-match")?.skipped,
).toBe(true);
expect(
result.checks.find((check) => check.rule === "planner-approach")
?.skipped,
).toBe(true);
});
});