Add skill governance and runtime routing
This commit is contained in:
1 parent
a900016be4
commit
221e9391e0
71 files changed
+7057
-102
No files matched your search
@@ -1,7 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import { judgeProcessRule } from "./process-judge.js";
|
||||
import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js";
|
||||
|
||||
function makeTrace(
|
||||
overrides?: Partial<ProcessTraceSnapshot["steps"]>,
|
||||
@@ -81,6 +86,11 @@ function makeTrace(
|
||||
}
|
||||
|
||||
describe("judgeProcessRule", () => {
|
||||
afterEach(() => {
|
||||
delete process.env["EVAL_PRM_ENABLED"];
|
||||
delete process.env["EVAL_PRM_WEIGHTS_PATH"];
|
||||
});
|
||||
|
||||
it("passes active process expectations", () => {
|
||||
const result = judgeProcessRule(
|
||||
{
|
||||
@@ -152,6 +162,33 @@ describe("judgeProcessRule", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("checks toolPlan step count against recorded tool calls", () => {
|
||||
const result = judgeProcessRule(
|
||||
{},
|
||||
makeTrace({
|
||||
plan: {
|
||||
name: "plan",
|
||||
startedAtMs: 3,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
approach: "validate-then-explain",
|
||||
mustAvoid: [],
|
||||
mustInclude: ["남은 절차"],
|
||||
suggestedTools: ["validate-nickname"],
|
||||
targetLengthChars: 120,
|
||||
toolPlanStepCount: 2,
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
result.checks.find((check) => check.rule === "tool-plan-consistent")
|
||||
?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
|
||||
it("fails tier and validator process regressions", () => {
|
||||
const result = judgeProcessRule(
|
||||
{},
|
||||
@@ -199,4 +236,37 @@ describe("judgeProcessRule", () => {
|
||||
?.skipped,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("adds PRM features, score, and attribution when enabled", () => {
|
||||
const weightsPath = join(mkdtempSync(join(tmpdir(), "prm-process-")), "weights.json");
|
||||
savePrmModel(weightsPath, {
|
||||
featureNames: [...PRM_FEATURE_NAMES],
|
||||
trainedAt: "2026-05-01T00:00:00.000Z",
|
||||
version: 1,
|
||||
weights: [1, 0, 0, 0, 0, 0, 0, 4, 0, 0],
|
||||
});
|
||||
process.env["EVAL_PRM_ENABLED"] = "true";
|
||||
process.env["EVAL_PRM_WEIGHTS_PATH"] = weightsPath;
|
||||
|
||||
const result = judgeProcessRule(
|
||||
{},
|
||||
makeTrace({
|
||||
validate: {
|
||||
name: "validate",
|
||||
startedAtMs: 6,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
status: "reject",
|
||||
reasons: ["stateConsistency"],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.prmFeatures).toMatchObject({
|
||||
validateStatus: 0,
|
||||
});
|
||||
expect(result.prmScore).toBeDefined();
|
||||
expect(result.rootCauseStep).toBe("validate");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user