Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent a900016be4
commit 221e9391e0
71 files changed
+7057 -102

No files matched your search

+71 -1
View File
@@ -1,7 +1,12 @@
import { describe, expect, it } from "vitest";
import { mkdtempSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { afterEach, describe, expect, it } from "vitest";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import { judgeProcessRule } from "./process-judge.js";
import { PRM_FEATURE_NAMES, savePrmModel } from "./prm.js";
function makeTrace(
overrides?: Partial<ProcessTraceSnapshot["steps"]>,
@@ -81,6 +86,11 @@ function makeTrace(
}
describe("judgeProcessRule", () => {
afterEach(() => {
delete process.env["EVAL_PRM_ENABLED"];
delete process.env["EVAL_PRM_WEIGHTS_PATH"];
});
it("passes active process expectations", () => {
const result = judgeProcessRule(
{
@@ -152,6 +162,33 @@ describe("judgeProcessRule", () => {
).toBe(false);
});
it("checks toolPlan step count against recorded tool calls", () => {
const result = judgeProcessRule(
{},
makeTrace({
plan: {
name: "plan",
startedAtMs: 3,
durationMs: 1,
output: {
approach: "validate-then-explain",
mustAvoid: [],
mustInclude: ["남은 절차"],
suggestedTools: ["validate-nickname"],
targetLengthChars: 120,
toolPlanStepCount: 2,
},
},
}),
);
expect(
result.checks.find((check) => check.rule === "tool-plan-consistent")
?.passed,
).toBe(false);
});
it("fails tier and validator process regressions", () => {
const result = judgeProcessRule(
{},
@@ -199,4 +236,37 @@ describe("judgeProcessRule", () => {
?.skipped,
).toBe(true);
});
it("adds PRM features, score, and attribution when enabled", () => {
const weightsPath = join(mkdtempSync(join(tmpdir(), "prm-process-")), "weights.json");
savePrmModel(weightsPath, {
featureNames: [...PRM_FEATURE_NAMES],
trainedAt: "2026-05-01T00:00:00.000Z",
version: 1,
weights: [1, 0, 0, 0, 0, 0, 0, 4, 0, 0],
});
process.env["EVAL_PRM_ENABLED"] = "true";
process.env["EVAL_PRM_WEIGHTS_PATH"] = weightsPath;
const result = judgeProcessRule(
{},
makeTrace({
validate: {
name: "validate",
startedAtMs: 6,
durationMs: 1,
output: {
status: "reject",
reasons: ["stateConsistency"],
},
},
}),
);
expect(result.prmFeatures).toMatchObject({
validateStatus: 0,
});
expect(result.prmScore).toBeDefined();
expect(result.rootCauseStep).toBe("validate");
});
});