Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,202 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import { judgeProcessRule } from "./process-judge.js";
|
||||
|
||||
function makeTrace(
|
||||
overrides?: Partial<ProcessTraceSnapshot["steps"]>,
|
||||
): ProcessTraceSnapshot {
|
||||
return {
|
||||
steps: {
|
||||
classify: {
|
||||
name: "classify",
|
||||
startedAtMs: 1,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
intent: "status-question",
|
||||
confidence: 0.9,
|
||||
source: "regex",
|
||||
},
|
||||
},
|
||||
derive: {
|
||||
name: "derive",
|
||||
startedAtMs: 2,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm-with-tools",
|
||||
shouldRespond: true,
|
||||
tier: 1,
|
||||
},
|
||||
},
|
||||
plan: {
|
||||
name: "plan",
|
||||
startedAtMs: 3,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
approach: "validate-then-explain",
|
||||
mustAvoid: [],
|
||||
mustInclude: ["남은 절차"],
|
||||
suggestedTools: ["validate-nickname"],
|
||||
targetLengthChars: 120,
|
||||
},
|
||||
},
|
||||
toolCalls: [
|
||||
{
|
||||
name: "tool-call",
|
||||
startedAtMs: 4,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
name: "validate-nickname",
|
||||
args: {
|
||||
nickname: "프리리/95/여/D",
|
||||
},
|
||||
ok: true,
|
||||
resultSummary: '{"valid":true}',
|
||||
},
|
||||
},
|
||||
],
|
||||
respond: {
|
||||
name: "respond",
|
||||
startedAtMs: 5,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
draftText: "방 하트 인증만 남았어요.",
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
},
|
||||
},
|
||||
validate: {
|
||||
name: "validate",
|
||||
startedAtMs: 6,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
status: "pass",
|
||||
reasons: [],
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("judgeProcessRule", () => {
|
||||
it("passes active process expectations", () => {
|
||||
const result = judgeProcessRule(
|
||||
{
|
||||
intent: "status-question",
|
||||
processExpected: {
|
||||
expectedApproach: "validate-then-explain",
|
||||
expectedTools: ["validate-nickname"],
|
||||
minSteps: 5,
|
||||
},
|
||||
},
|
||||
makeTrace(),
|
||||
);
|
||||
|
||||
expect(result.score).toBe(1);
|
||||
expect(result.totalCount).toBeGreaterThan(0);
|
||||
expect(result.checks.every((check) => check.skipped || check.passed)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("fails intent, tool, and approach mismatches", () => {
|
||||
const result = judgeProcessRule(
|
||||
{
|
||||
intent: "photo-question",
|
||||
processExpected: {
|
||||
expectedApproach: "answer-direct",
|
||||
expectedTools: ["check-onboarding-progress"],
|
||||
toolMode: "exact",
|
||||
},
|
||||
},
|
||||
makeTrace(),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "intent-match")?.passed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "planner-approach")
|
||||
?.passed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "tools-selected-correctly")
|
||||
?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("fails invalid tool arguments", () => {
|
||||
const result = judgeProcessRule(
|
||||
{},
|
||||
makeTrace({
|
||||
toolCalls: [
|
||||
{
|
||||
name: "tool-call",
|
||||
startedAtMs: 4,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
name: "validate-nickname",
|
||||
args: {},
|
||||
ok: false,
|
||||
resultSummary: "invalid",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "tool-args-valid")?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("fails tier and validator process regressions", () => {
|
||||
const result = judgeProcessRule(
|
||||
{},
|
||||
makeTrace({
|
||||
derive: {
|
||||
name: "derive",
|
||||
startedAtMs: 2,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm-with-tools",
|
||||
shouldRespond: true,
|
||||
tier: 0,
|
||||
},
|
||||
},
|
||||
validate: {
|
||||
name: "validate",
|
||||
startedAtMs: 6,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
status: "reject",
|
||||
reasons: ["stateConsistency"],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "tier-appropriate")?.passed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "validate-no-reject")
|
||||
?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("skips optional expectations when a golden case has not opted in", () => {
|
||||
const result = judgeProcessRule({}, makeTrace());
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "intent-match")?.skipped,
|
||||
).toBe(true);
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "planner-approach")
|
||||
?.skipped,
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user