206 lines
4.9 KiB
TypeScript
206 lines
4.9 KiB
TypeScript
import { mkdtempSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
|
|
import {
|
|
buildPrmTrainingSamples,
|
|
extractPrmFeatures,
|
|
loadPrmModel,
|
|
predictPrmScore,
|
|
savePrmModel,
|
|
trainRidgePrmModel,
|
|
} from "./prm.js";
|
|
import type { EvalArchiveEntry } from "../archive.js";
|
|
import type { ReviewLabel } from "../review/queue.js";
|
|
|
|
function trace(overrides?: Partial<ProcessTraceSnapshot["steps"]>): ProcessTraceSnapshot {
|
|
return {
|
|
steps: {
|
|
classify: {
|
|
name: "classify",
|
|
startedAtMs: 1,
|
|
durationMs: 1,
|
|
output: {
|
|
confidence: 0.8,
|
|
intent: "photo-question",
|
|
source: "llm",
|
|
},
|
|
},
|
|
derive: {
|
|
name: "derive",
|
|
startedAtMs: 2,
|
|
durationMs: 1,
|
|
output: {
|
|
phase: "photo-pending",
|
|
responseMode: "llm",
|
|
shouldRespond: true,
|
|
tier: 1,
|
|
},
|
|
},
|
|
plan: {
|
|
name: "plan",
|
|
startedAtMs: 3,
|
|
durationMs: 1,
|
|
output: {
|
|
approach: "answer-direct",
|
|
mustAvoid: [],
|
|
mustInclude: [],
|
|
suggestedTools: [],
|
|
targetLengthChars: 120,
|
|
},
|
|
},
|
|
validate: {
|
|
name: "validate",
|
|
startedAtMs: 4,
|
|
durationMs: 1,
|
|
output: {
|
|
status: "pass",
|
|
reasons: [],
|
|
},
|
|
},
|
|
...overrides,
|
|
},
|
|
};
|
|
}
|
|
|
|
function archiveEntry(streamId: string, processTrace: ProcessTraceSnapshot): EvalArchiveEntry {
|
|
return {
|
|
streamId,
|
|
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
|
|
request: {
|
|
requestId: streamId,
|
|
skill: "onboarding",
|
|
chatroomId: "room-1",
|
|
userId: "user-1",
|
|
nickname: "닉",
|
|
trigger: "user-message",
|
|
},
|
|
response: {
|
|
requestId: streamId,
|
|
text: "응답",
|
|
confidence: 0.9,
|
|
source: "llm",
|
|
},
|
|
derived: {
|
|
phase: { kind: "photo-pending" },
|
|
intent: { kind: "photo-question" },
|
|
shouldRespond: true,
|
|
responseMode: "llm",
|
|
suggestedMaxTokens: 128,
|
|
allowShortResponse: true,
|
|
tier: 1,
|
|
},
|
|
classified: {
|
|
intent: { kind: "photo-question" },
|
|
confidence: 0.8,
|
|
source: "llm",
|
|
},
|
|
validation: {
|
|
status: "pass",
|
|
text: "응답",
|
|
reasons: [],
|
|
},
|
|
llmJudge: {
|
|
skipped: true,
|
|
reason: "disabled",
|
|
},
|
|
latencyMs: 100,
|
|
process: processTrace,
|
|
tier: 1,
|
|
};
|
|
}
|
|
|
|
describe("PRM", () => {
|
|
it("extracts stable feature vectors from process traces", () => {
|
|
const features = extractPrmFeatures(
|
|
trace({
|
|
toolCalls: [
|
|
{
|
|
name: "tool-call",
|
|
startedAtMs: 5,
|
|
durationMs: 1,
|
|
output: {
|
|
args: {},
|
|
errorKind: "permanent",
|
|
name: "lookup-community-rule",
|
|
ok: false,
|
|
resultSummary: "boom",
|
|
},
|
|
},
|
|
],
|
|
}),
|
|
);
|
|
|
|
expect(features).toMatchObject({
|
|
classifyConfidence: 0.8,
|
|
classifySource: 1,
|
|
planApproach: "answer-direct",
|
|
tier: 1,
|
|
toolCallCount: 1,
|
|
toolErrorCount: 1,
|
|
validateStatus: 2,
|
|
});
|
|
});
|
|
|
|
it("trains, saves, loads, and predicts a ridge model", () => {
|
|
const samples = [
|
|
{ features: extractPrmFeatures(trace()), target: 5 },
|
|
{
|
|
features: extractPrmFeatures(
|
|
trace({
|
|
validate: {
|
|
name: "validate",
|
|
startedAtMs: 4,
|
|
durationMs: 1,
|
|
output: {
|
|
status: "reject",
|
|
reasons: ["stateConsistency"],
|
|
},
|
|
},
|
|
}),
|
|
),
|
|
target: 1,
|
|
},
|
|
];
|
|
const model = trainRidgePrmModel(samples, {
|
|
lambda: 0.1,
|
|
trainedAt: "2026-05-01T00:00:00.000Z",
|
|
});
|
|
const path = join(mkdtempSync(join(tmpdir(), "prm-")), "weights.json");
|
|
|
|
savePrmModel(path, model);
|
|
const loaded = loadPrmModel(path);
|
|
const goodSample = samples[0];
|
|
const badSample = samples[1];
|
|
|
|
expect(loaded?.featureNames).toEqual(model.featureNames);
|
|
expect(goodSample).toBeDefined();
|
|
expect(badSample).toBeDefined();
|
|
expect(predictPrmScore(model, goodSample!.features)).toBeGreaterThan(
|
|
predictPrmScore(model, badSample!.features),
|
|
);
|
|
});
|
|
|
|
it("builds training samples by joining labels to archived traces", () => {
|
|
const labels: ReviewLabel[] = [
|
|
{
|
|
evalStreamId: "1-0",
|
|
humanScore: 4,
|
|
rationale: "ok",
|
|
reviewedAt: "2026-05-01T00:00:00.000Z",
|
|
reviewedBy: "tester",
|
|
verdict: "pass",
|
|
},
|
|
];
|
|
const samples = buildPrmTrainingSamples(labels, [
|
|
archiveEntry("1-0", trace()),
|
|
]);
|
|
|
|
expect(samples).toHaveLength(1);
|
|
expect(samples[0]?.target).toBe(4);
|
|
});
|
|
});
|