250 lines
6.8 KiB
TypeScript
250 lines
6.8 KiB
TypeScript
import type {
|
|
ProcessTraceSnapshot,
|
|
ProcessTraceStepName,
|
|
} from "@skills/onboarding/runtime/trace";
|
|
import type { EvalArchiveEntry, MongoConnectionLike } from "../archive.js";
|
|
import type { SliceKey } from "../aggregator.js";
|
|
import type { PrmModel, PrmFeatureVector } from "./prm.js";
|
|
import {
|
|
extractPrmFeatures,
|
|
predictPrmScore,
|
|
} from "./prm.js";
|
|
|
|
export const EVAL_PROCESS_ATTRIBUTION_COLLECTION =
|
|
"eval_process_attribution";
|
|
|
|
export interface TraceAttributionResult {
|
|
rootCauseScoreLoss: number;
|
|
rootCauseStep: ProcessTraceStepName | null;
|
|
stepLosses: Partial<Record<ProcessTraceStepName, number>>;
|
|
}
|
|
|
|
export interface ProcessAttributionEntry {
|
|
createdAt: Date;
|
|
evaluatedAt: Date;
|
|
modelTrainedAt: string;
|
|
prmFeatures: PrmFeatureVector;
|
|
prmScore: number;
|
|
rootCauseScoreLoss: number;
|
|
rootCauseStep: ProcessTraceStepName | null;
|
|
slice: SliceKey;
|
|
stepLosses: Partial<Record<ProcessTraceStepName, number>>;
|
|
streamId: string;
|
|
}
|
|
|
|
interface BulkWriteResultLike {
|
|
modifiedCount?: number | undefined;
|
|
upsertedCount?: number | undefined;
|
|
}
|
|
|
|
interface ProcessAttributionCollectionLike {
|
|
bulkWrite(
|
|
operations: unknown[],
|
|
options?: { ordered?: boolean | undefined },
|
|
): Promise<BulkWriteResultLike>;
|
|
createIndex(
|
|
keys: Record<string, unknown>,
|
|
options?: Record<string, unknown>,
|
|
): Promise<unknown>;
|
|
}
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(6));
|
|
}
|
|
|
|
function counterfactualForStep(
|
|
features: PrmFeatureVector,
|
|
step: ProcessTraceStepName,
|
|
): PrmFeatureVector {
|
|
switch (step) {
|
|
case "classify":
|
|
return {
|
|
...features,
|
|
abstentionScore: 0,
|
|
classifyConfidence: 1,
|
|
classifySource: 0,
|
|
};
|
|
case "derive":
|
|
return {
|
|
...features,
|
|
tier: 0,
|
|
};
|
|
case "plan":
|
|
return {
|
|
...features,
|
|
planApproach: "answer-direct",
|
|
};
|
|
case "tool-call":
|
|
return {
|
|
...features,
|
|
toolCallCount: 0,
|
|
toolErrorCount: 0,
|
|
};
|
|
case "critic":
|
|
return {
|
|
...features,
|
|
criticDecision: 0,
|
|
};
|
|
case "validate":
|
|
return {
|
|
...features,
|
|
validateStatus: 2,
|
|
};
|
|
case "respond":
|
|
return features;
|
|
}
|
|
}
|
|
|
|
function recordedSteps(trace: ProcessTraceSnapshot): ProcessTraceStepName[] {
|
|
const steps: ProcessTraceStepName[] = [];
|
|
if (trace.steps.classify) steps.push("classify");
|
|
if (trace.steps.derive) steps.push("derive");
|
|
if (trace.steps.plan) steps.push("plan");
|
|
if ((trace.steps.toolCalls?.length ?? 0) > 0) steps.push("tool-call");
|
|
if (trace.steps.respond) steps.push("respond");
|
|
if (trace.steps.critic) steps.push("critic");
|
|
if (trace.steps.validate) steps.push("validate");
|
|
return steps;
|
|
}
|
|
|
|
export function attributeTraceWithPrm(
|
|
trace: ProcessTraceSnapshot,
|
|
model: PrmModel,
|
|
): TraceAttributionResult {
|
|
const features = extractPrmFeatures(trace);
|
|
const original = predictPrmScore(model, features);
|
|
const stepLosses: Partial<Record<ProcessTraceStepName, number>> = {};
|
|
|
|
for (const step of recordedSteps(trace)) {
|
|
const counterfactual = counterfactualForStep(features, step);
|
|
const score = predictPrmScore(model, counterfactual);
|
|
stepLosses[step] = round(score - original);
|
|
}
|
|
|
|
const [rootCauseStep, rootCauseScoreLoss] =
|
|
(Object.entries(stepLosses) as Array<[ProcessTraceStepName, number]>)
|
|
.filter(([, loss]) => loss > 0)
|
|
.sort((left, right) => right[1] - left[1])[0] ?? [null, 0];
|
|
|
|
return {
|
|
rootCauseScoreLoss: round(rootCauseScoreLoss),
|
|
rootCauseStep,
|
|
stepLosses,
|
|
};
|
|
}
|
|
|
|
function sliceFromEntry(entry: EvalArchiveEntry): SliceKey {
|
|
const abstention = (entry.process as ProcessTraceSnapshot | undefined)
|
|
?.steps.classify?.output.abstention;
|
|
|
|
return {
|
|
tier: entry.tier,
|
|
intent: entry.derived.intent.kind,
|
|
phase: entry.derived.phase.kind,
|
|
...(entry.derived.routingDecision
|
|
? { routingDecision: entry.derived.routingDecision.kind }
|
|
: {}),
|
|
...(abstention?.action.kind !== undefined &&
|
|
abstention.action.kind !== "proceed"
|
|
? { abstentionAction: abstention.action.kind }
|
|
: {}),
|
|
...(abstention?.reason ? { abstentionReason: abstention.reason } : {}),
|
|
};
|
|
}
|
|
|
|
async function ensureProcessAttributionIndexes(
|
|
collection: ProcessAttributionCollectionLike,
|
|
): Promise<void> {
|
|
await Promise.all([
|
|
collection.createIndex({ streamId: 1 }, { unique: true }),
|
|
collection.createIndex({ evaluatedAt: -1 }),
|
|
collection.createIndex({ rootCauseStep: 1, rootCauseScoreLoss: -1 }),
|
|
collection.createIndex({
|
|
"slice.tier": 1,
|
|
"slice.intent": 1,
|
|
"slice.phase": 1,
|
|
evaluatedAt: -1,
|
|
}),
|
|
]);
|
|
}
|
|
|
|
export function getProcessAttributionCollection(
|
|
mongoConnection: MongoConnectionLike,
|
|
): ProcessAttributionCollectionLike {
|
|
return mongoConnection.collection(
|
|
EVAL_PROCESS_ATTRIBUTION_COLLECTION,
|
|
) as ProcessAttributionCollectionLike;
|
|
}
|
|
|
|
export function buildProcessAttributionEntries(
|
|
archiveEntries: EvalArchiveEntry[],
|
|
model: PrmModel,
|
|
options?: { nowMs?: number | undefined },
|
|
): ProcessAttributionEntry[] {
|
|
const createdAt = new Date(options?.nowMs ?? Date.now());
|
|
|
|
return archiveEntries.flatMap((entry) => {
|
|
const trace = entry.process as ProcessTraceSnapshot | undefined;
|
|
if (!trace) {
|
|
return [];
|
|
}
|
|
|
|
const prmFeatures = extractPrmFeatures(trace);
|
|
const attribution = attributeTraceWithPrm(trace, model);
|
|
return [
|
|
{
|
|
createdAt,
|
|
evaluatedAt: entry.evaluatedAt,
|
|
modelTrainedAt: model.trainedAt,
|
|
prmFeatures,
|
|
prmScore: predictPrmScore(model, prmFeatures),
|
|
rootCauseScoreLoss: attribution.rootCauseScoreLoss,
|
|
rootCauseStep: attribution.rootCauseStep,
|
|
slice: sliceFromEntry(entry),
|
|
stepLosses: attribution.stepLosses,
|
|
streamId: entry.streamId,
|
|
},
|
|
];
|
|
});
|
|
}
|
|
|
|
export async function writeProcessAttributions(
|
|
mongoConnection: MongoConnectionLike,
|
|
archiveEntries: EvalArchiveEntry[],
|
|
model: PrmModel,
|
|
options?: { nowMs?: number | undefined },
|
|
): Promise<{
|
|
attributedCount: number;
|
|
modifiedCount: number;
|
|
upsertedCount: number;
|
|
}> {
|
|
const collection = getProcessAttributionCollection(mongoConnection);
|
|
await ensureProcessAttributionIndexes(collection);
|
|
|
|
const entries = buildProcessAttributionEntries(archiveEntries, model, options);
|
|
if (entries.length === 0) {
|
|
return {
|
|
attributedCount: 0,
|
|
modifiedCount: 0,
|
|
upsertedCount: 0,
|
|
};
|
|
}
|
|
|
|
const result = await collection.bulkWrite(
|
|
entries.map((entry) => ({
|
|
updateOne: {
|
|
filter: { streamId: entry.streamId },
|
|
update: { $set: entry },
|
|
upsert: true,
|
|
},
|
|
})),
|
|
{ ordered: false },
|
|
);
|
|
|
|
return {
|
|
attributedCount: entries.length,
|
|
modifiedCount: result.modifiedCount ?? 0,
|
|
upsertedCount: result.upsertedCount ?? 0,
|
|
};
|
|
}
|