Files
lolympus-agent/src/eval/judges/prm.test.ts
T

206 lines
4.9 KiB
TypeScript

import { mkdtempSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { describe, expect, it } from "vitest";
import type { ProcessTraceSnapshot } from "@skills/onboarding/runtime/trace";
import {
buildPrmTrainingSamples,
extractPrmFeatures,
loadPrmModel,
predictPrmScore,
savePrmModel,
trainRidgePrmModel,
} from "./prm.js";
import type { EvalArchiveEntry } from "../archive.js";
import type { ReviewLabel } from "../review/queue.js";
function trace(overrides?: Partial<ProcessTraceSnapshot["steps"]>): ProcessTraceSnapshot {
return {
steps: {
classify: {
name: "classify",
startedAtMs: 1,
durationMs: 1,
output: {
confidence: 0.8,
intent: "photo-question",
source: "llm",
},
},
derive: {
name: "derive",
startedAtMs: 2,
durationMs: 1,
output: {
phase: "photo-pending",
responseMode: "llm",
shouldRespond: true,
tier: 1,
},
},
plan: {
name: "plan",
startedAtMs: 3,
durationMs: 1,
output: {
approach: "answer-direct",
mustAvoid: [],
mustInclude: [],
suggestedTools: [],
targetLengthChars: 120,
},
},
validate: {
name: "validate",
startedAtMs: 4,
durationMs: 1,
output: {
status: "pass",
reasons: [],
},
},
...overrides,
},
};
}
function archiveEntry(streamId: string, processTrace: ProcessTraceSnapshot): EvalArchiveEntry {
return {
streamId,
evaluatedAt: new Date("2026-05-01T00:00:00.000Z"),
request: {
requestId: streamId,
skill: "onboarding",
chatroomId: "room-1",
userId: "user-1",
nickname: "닉",
trigger: "user-message",
},
response: {
requestId: streamId,
text: "응답",
confidence: 0.9,
source: "llm",
},
derived: {
phase: { kind: "photo-pending" },
intent: { kind: "photo-question" },
shouldRespond: true,
responseMode: "llm",
suggestedMaxTokens: 128,
allowShortResponse: true,
tier: 1,
},
classified: {
intent: { kind: "photo-question" },
confidence: 0.8,
source: "llm",
},
validation: {
status: "pass",
text: "응답",
reasons: [],
},
llmJudge: {
skipped: true,
reason: "disabled",
},
latencyMs: 100,
process: processTrace,
tier: 1,
};
}
describe("PRM", () => {
it("extracts stable feature vectors from process traces", () => {
const features = extractPrmFeatures(
trace({
toolCalls: [
{
name: "tool-call",
startedAtMs: 5,
durationMs: 1,
output: {
args: {},
errorKind: "permanent",
name: "lookup-community-rule",
ok: false,
resultSummary: "boom",
},
},
],
}),
);
expect(features).toMatchObject({
classifyConfidence: 0.8,
classifySource: 1,
planApproach: "answer-direct",
tier: 1,
toolCallCount: 1,
toolErrorCount: 1,
validateStatus: 2,
});
});
it("trains, saves, loads, and predicts a ridge model", () => {
const samples = [
{ features: extractPrmFeatures(trace()), target: 5 },
{
features: extractPrmFeatures(
trace({
validate: {
name: "validate",
startedAtMs: 4,
durationMs: 1,
output: {
status: "reject",
reasons: ["stateConsistency"],
},
},
}),
),
target: 1,
},
];
const model = trainRidgePrmModel(samples, {
lambda: 0.1,
trainedAt: "2026-05-01T00:00:00.000Z",
});
const path = join(mkdtempSync(join(tmpdir(), "prm-")), "weights.json");
savePrmModel(path, model);
const loaded = loadPrmModel(path);
const goodSample = samples[0];
const badSample = samples[1];
expect(loaded?.featureNames).toEqual(model.featureNames);
expect(goodSample).toBeDefined();
expect(badSample).toBeDefined();
expect(predictPrmScore(model, goodSample!.features)).toBeGreaterThan(
predictPrmScore(model, badSample!.features),
);
});
it("builds training samples by joining labels to archived traces", () => {
const labels: ReviewLabel[] = [
{
evalStreamId: "1-0",
humanScore: 4,
rationale: "ok",
reviewedAt: "2026-05-01T00:00:00.000Z",
reviewedBy: "tester",
verdict: "pass",
},
];
const samples = buildPrmTrainingSamples(labels, [
archiveEntry("1-0", trace()),
]);
expect(samples).toHaveLength(1);
expect(samples[0]?.target).toBe(4);
});
});