Add skill governance and runtime routing
This commit is contained in:
1 parent
6f09408423
commit
41dcfcdef7
71 files changed
+7057
-102
No files matched your search
@@ -0,0 +1,205 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
||||
import {
|
||||
buildPrmTrainingSamples,
|
||||
extractPrmFeatures,
|
||||
loadPrmModel,
|
||||
predictPrmScore,
|
||||
savePrmModel,
|
||||
trainRidgePrmModel,
|
||||
} from "./prm.js";
|
||||
import type { EvalArchiveEntry } from "../archive.js";
|
||||
import type { ReviewLabel } from "../review/queue.js";
|
||||
|
||||
function trace(overrides?: Partial<ProcessTraceSnapshot["steps"]>): ProcessTraceSnapshot {
|
||||
return {
|
||||
steps: {
|
||||
classify: {
|
||||
name: "classify",
|
||||
startedAtMs: 1,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
confidence: 0.8,
|
||||
intent: "photo-question",
|
||||
source: "llm",
|
||||
},
|
||||
},
|
||||
derive: {
|
||||
name: "derive",
|
||||
startedAtMs: 2,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
tier: 1,
|
||||
},
|
||||
},
|
||||
plan: {
|
||||
name: "plan",
|
||||
startedAtMs: 3,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
approach: "answer-direct",
|
||||
mustAvoid: [],
|
||||
mustInclude: [],
|
||||
suggestedTools: [],
|
||||
targetLengthChars: 120,
|
||||
},
|
||||
},
|
||||
validate: {
|
||||
name: "validate",
|
||||
startedAtMs: 4,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
status: "pass",
|
||||
reasons: [],
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function archiveEntry(streamId: string, processTrace: ProcessTraceSnapshot): EvalArchiveEntry {
|
||||
return {
|
||||
streamId,
|
||||
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
||||
request: {
|
||||
requestId: streamId,
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "닉",
|
||||
trigger: "user-message",
|
||||
},
|
||||
response: {
|
||||
requestId: streamId,
|
||||
text: "응답",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
},
|
||||
derived: {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "photo-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 1,
|
||||
},
|
||||
classified: {
|
||||
intent: { kind: "photo-question" },
|
||||
confidence: 0.8,
|
||||
source: "llm",
|
||||
},
|
||||
validation: {
|
||||
status: "pass",
|
||||
text: "응답",
|
||||
reasons: [],
|
||||
},
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
reason: "disabled",
|
||||
},
|
||||
latencyMs: 100,
|
||||
process: processTrace,
|
||||
tier: 1,
|
||||
};
|
||||
}
|
||||
|
||||
describe("PRM", () => {
|
||||
it("extracts stable feature vectors from process traces", () => {
|
||||
const features = extractPrmFeatures(
|
||||
trace({
|
||||
toolCalls: [
|
||||
{
|
||||
name: "tool-call",
|
||||
startedAtMs: 5,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
args: {},
|
||||
errorKind: "permanent",
|
||||
name: "lookup-community-rule",
|
||||
ok: false,
|
||||
resultSummary: "boom",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(features).toMatchObject({
|
||||
classifyConfidence: 0.8,
|
||||
classifySource: 1,
|
||||
planApproach: "answer-direct",
|
||||
tier: 1,
|
||||
toolCallCount: 1,
|
||||
toolErrorCount: 1,
|
||||
validateStatus: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("trains, saves, loads, and predicts a ridge model", () => {
|
||||
const samples = [
|
||||
{ features: extractPrmFeatures(trace()), target: 5 },
|
||||
{
|
||||
features: extractPrmFeatures(
|
||||
trace({
|
||||
validate: {
|
||||
name: "validate",
|
||||
startedAtMs: 4,
|
||||
durationMs: 1,
|
||||
output: {
|
||||
status: "reject",
|
||||
reasons: ["stateConsistency"],
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
target: 1,
|
||||
},
|
||||
];
|
||||
const model = trainRidgePrmModel(samples, {
|
||||
lambda: 0.1,
|
||||
trainedAt: "2026-05-01T00:00:00.000Z",
|
||||
});
|
||||
const path = join(mkdtempSync(join(tmpdir(), "prm-")), "weights.json");
|
||||
|
||||
savePrmModel(path, model);
|
||||
const loaded = loadPrmModel(path);
|
||||
const goodSample = samples[0];
|
||||
const badSample = samples[1];
|
||||
|
||||
expect(loaded?.featureNames).toEqual(model.featureNames);
|
||||
expect(goodSample).toBeDefined();
|
||||
expect(badSample).toBeDefined();
|
||||
expect(predictPrmScore(model, goodSample!.features)).toBeGreaterThan(
|
||||
predictPrmScore(model, badSample!.features),
|
||||
);
|
||||
});
|
||||
|
||||
it("builds training samples by joining labels to archived traces", () => {
|
||||
const labels: ReviewLabel[] = [
|
||||
{
|
||||
evalStreamId: "1-0",
|
||||
humanScore: 4,
|
||||
rationale: "ok",
|
||||
reviewedAt: "2026-05-01T00:00:00.000Z",
|
||||
reviewedBy: "tester",
|
||||
verdict: "pass",
|
||||
},
|
||||
];
|
||||
const samples = buildPrmTrainingSamples(labels, [
|
||||
archiveEntry("1-0", trace()),
|
||||
]);
|
||||
|
||||
expect(samples).toHaveLength(1);
|
||||
expect(samples[0]?.target).toBe(4);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user