import { mkdtempSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { describe, expect, it } from "vitest"; import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace"; import { buildPrmTrainingSamples, extractPrmFeatures, loadPrmModel, predictPrmScore, savePrmModel, trainRidgePrmModel, } from "./prm.js"; import type { EvalArchiveEntry } from "../archive.js"; import type { ReviewLabel } from "../review/queue.js"; function trace(overrides?: Partial): ProcessTraceSnapshot { return { steps: { classify: { name: "classify", startedAtMs: 1, durationMs: 1, output: { confidence: 0.8, intent: "photo-question", source: "llm", }, }, derive: { name: "derive", startedAtMs: 2, durationMs: 1, output: { phase: "photo-pending", responseMode: "llm", shouldRespond: true, tier: 1, }, }, plan: { name: "plan", startedAtMs: 3, durationMs: 1, output: { approach: "answer-direct", mustAvoid: [], mustInclude: [], suggestedTools: [], targetLengthChars: 120, }, }, validate: { name: "validate", startedAtMs: 4, durationMs: 1, output: { status: "pass", reasons: [], }, }, ...overrides, }, }; } function archiveEntry(streamId: string, processTrace: ProcessTraceSnapshot): EvalArchiveEntry { return { streamId, evaluatedAt: new Date("2026-05-01T00:00:00.000Z"), request: { requestId: streamId, skill: "onboarding", chatroomId: "room-1", userId: "user-1", nickname: "닉", trigger: "user-message", }, response: { requestId: streamId, text: "응답", confidence: 0.9, source: "llm", }, derived: { phase: { kind: "photo-pending" }, intent: { kind: "photo-question" }, shouldRespond: true, responseMode: "llm", suggestedMaxTokens: 128, allowShortResponse: true, tier: 1, }, classified: { intent: { kind: "photo-question" }, confidence: 0.8, source: "llm", }, validation: { status: "pass", text: "응답", reasons: [], }, llmJudge: { skipped: true, reason: "disabled", }, latencyMs: 100, process: processTrace, tier: 1, }; } describe("PRM", () => { it("extracts stable feature vectors from process traces", () => { const features = extractPrmFeatures( trace({ toolCalls: [ { name: "tool-call", startedAtMs: 5, durationMs: 1, output: { args: {}, errorKind: "permanent", name: "lookup-community-rule", ok: false, resultSummary: "boom", }, }, ], }), ); expect(features).toMatchObject({ classifyConfidence: 0.8, classifySource: 1, planApproach: "answer-direct", tier: 1, toolCallCount: 1, toolErrorCount: 1, validateStatus: 2, }); }); it("trains, saves, loads, and predicts a ridge model", () => { const samples = [ { features: extractPrmFeatures(trace()), target: 5 }, { features: extractPrmFeatures( trace({ validate: { name: "validate", startedAtMs: 4, durationMs: 1, output: { status: "reject", reasons: ["stateConsistency"], }, }, }), ), target: 1, }, ]; const model = trainRidgePrmModel(samples, { lambda: 0.1, trainedAt: "2026-05-01T00:00:00.000Z", }); const path = join(mkdtempSync(join(tmpdir(), "prm-")), "weights.json"); savePrmModel(path, model); const loaded = loadPrmModel(path); const goodSample = samples[0]; const badSample = samples[1]; expect(loaded?.featureNames).toEqual(model.featureNames); expect(goodSample).toBeDefined(); expect(badSample).toBeDefined(); expect(predictPrmScore(model, goodSample!.features)).toBeGreaterThan( predictPrmScore(model, badSample!.features), ); }); it("builds training samples by joining labels to archived traces", () => { const labels: ReviewLabel[] = [ { evalStreamId: "1-0", humanScore: 4, rationale: "ok", reviewedAt: "2026-05-01T00:00:00.000Z", reviewedBy: "tester", verdict: "pass", }, ]; const samples = buildPrmTrainingSamples(labels, [ archiveEntry("1-0", trace()), ]); expect(samples).toHaveLength(1); expect(samples[0]?.target).toBe(4); }); });