import type { ProcessTraceSnapshot, ProcessTraceStepName, } from "@skills/onboarding/runtime/trace"; import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js"; import type { SliceKey } from "../aggregator.js"; import type { PrmModel, PrmFeatureVector } from "./prm.js"; import { extractPrmFeatures, predictPrmScore, } from "./prm.js"; export const EVAL_PROCESS_ATTRIBUTION_COLLECTION = "eval_process_attribution"; export interface TraceAttributionResult { rootCauseScoreLoss: number; rootCauseStep: ProcessTraceStepName | null; stepLosses: Partial>; } export interface ProcessAttributionEntry { createdAt: Date; evaluatedAt: Date; modelTrainedAt: string; prmFeatures: PrmFeatureVector; prmScore: number; rootCauseScoreLoss: number; rootCauseStep: ProcessTraceStepName | null; slice: SliceKey; stepLosses: Partial>; streamId: string; } interface BulkWriteResultLike { modifiedCount?: number | undefined; upsertedCount?: number | undefined; } interface ProcessAttributionCollectionLike { bulkWrite( operations: unknown[], options?: { ordered?: boolean | undefined }, ): Promise; createIndex( keys: Record, options?: Record, ): Promise; } function round(value: number): number { return Number(value.toFixed(6)); } function counterfactualForStep( features: PrmFeatureVector, step: ProcessTraceStepName, ): PrmFeatureVector { switch (step) { case "classify": return { ...features, abstentionScore: 0, classifyConfidence: 1, classifySource: 0, }; case "derive": return { ...features, tier: 0, }; case "plan": return { ...features, planApproach: "answer-direct", }; case "tool-call": return { ...features, toolCallCount: 0, toolErrorCount: 0, }; case "critic": return { ...features, criticDecision: 0, }; case "validate": return { ...features, validateStatus: 2, }; case "respond": return features; } } function recordedSteps(trace: ProcessTraceSnapshot): ProcessTraceStepName[] { const steps: ProcessTraceStepName[] = []; if (trace.steps.classify) steps.push("classify"); if (trace.steps.derive) steps.push("derive"); if (trace.steps.plan) steps.push("plan"); if ((trace.steps.toolCalls?.length ?? 0) > 0) steps.push("tool-call"); if (trace.steps.respond) steps.push("respond"); if (trace.steps.critic) steps.push("critic"); if (trace.steps.validate) steps.push("validate"); return steps; } export function attributeTraceWithPrm( trace: ProcessTraceSnapshot, model: PrmModel, ): TraceAttributionResult { const features = extractPrmFeatures(trace); const original = predictPrmScore(model, features); const stepLosses: Partial> = {}; for (const step of recordedSteps(trace)) { const counterfactual = counterfactualForStep(features, step); const score = predictPrmScore(model, counterfactual); stepLosses[step] = round(score - original); } const [rootCauseStep, rootCauseScoreLoss] = (Object.entries(stepLosses) as Array<[ProcessTraceStepName, number]>) .filter(([, loss]) => loss > 0) .sort((left, right) => right[1] - left[1])[0] ?? [null, 0]; return { rootCauseScoreLoss: round(rootCauseScoreLoss), rootCauseStep, stepLosses, }; } function sliceFromEntry(entry: EvalArchiveEntry): SliceKey { const abstention = (entry.process as ProcessTraceSnapshot | undefined) ?.steps.classify?.output.abstention; return { tier: entry.tier, intent: entry.derived.intent.kind, phase: entry.derived.phase.kind, ...(entry.derived.routingDecision ? { routingDecision: entry.derived.routingDecision.kind } : {}), ...(abstention?.action.kind !== undefined && abstention.action.kind !== "proceed" ? { abstentionAction: abstention.action.kind } : {}), ...(abstention?.reason ? { abstentionReason: abstention.reason } : {}), }; } async function ensureProcessAttributionIndexes( collection: ProcessAttributionCollectionLike, ): Promise { await Promise.all([ collection.createIndex({ streamId: 1 }, { unique: true }), collection.createIndex({ evaluatedAt: -1 }), collection.createIndex({ rootCauseStep: 1, rootCauseScoreLoss: -1 }), collection.createIndex({ "slice.tier": 1, "slice.intent": 1, "slice.phase": 1, evaluatedAt: -1, }), ]); } export function getProcessAttributionCollection( mongoConnection: MongoConnectionLike, ): ProcessAttributionCollectionLike { return mongoConnection.collection( EVAL_PROCESS_ATTRIBUTION_COLLECTION, ) as ProcessAttributionCollectionLike; } export function buildProcessAttributionEntries( archiveEntries: EvalArchiveEntry[], model: PrmModel, options?: { nowMs?: number | undefined }, ): ProcessAttributionEntry[] { const createdAt = new Date(options?.nowMs ?? Date.now()); return archiveEntries.flatMap((entry) => { const trace = entry.process as ProcessTraceSnapshot | undefined; if (!trace) { return []; } const prmFeatures = extractPrmFeatures(trace); const attribution = attributeTraceWithPrm(trace, model); return [ { createdAt, evaluatedAt: entry.evaluatedAt, modelTrainedAt: model.trainedAt, prmFeatures, prmScore: predictPrmScore(model, prmFeatures), rootCauseScoreLoss: attribution.rootCauseScoreLoss, rootCauseStep: attribution.rootCauseStep, slice: sliceFromEntry(entry), stepLosses: attribution.stepLosses, streamId: entry.streamId, }, ]; }); } export async function writeProcessAttributions( mongoConnection: MongoConnectionLike, archiveEntries: EvalArchiveEntry[], model: PrmModel, options?: { nowMs?: number | undefined }, ): Promise<{ attributedCount: number; modifiedCount: number; upsertedCount: number; }> { const collection = getProcessAttributionCollection(mongoConnection); await ensureProcessAttributionIndexes(collection); const entries = buildProcessAttributionEntries(archiveEntries, model, options); if (entries.length === 0) { return { attributedCount: 0, modifiedCount: 0, upsertedCount: 0, }; } const result = await collection.bulkWrite( entries.map((entry) => ({ updateOne: { filter: { streamId: entry.streamId }, update: { $set: entry }, upsert: true, }, })), { ordered: false }, ); return { attributedCount: entries.length, modifiedCount: result.modifiedCount ?? 0, upsertedCount: result.upsertedCount ?? 0, }; }