Add evaluation operations and reporting
This commit is contained in:
1 parent
60f33b8f4b
commit
a900016be4
60 files changed
+8689
-142
No files matched your search
@@ -0,0 +1,169 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
|
||||
export type ProcessTraceStepName =
|
||||
| "classify"
|
||||
| "critic"
|
||||
| "derive"
|
||||
| "plan"
|
||||
| "respond"
|
||||
| "tool-call"
|
||||
| "validate";
|
||||
|
||||
export interface ProcessTraceStep<T> {
|
||||
name: ProcessTraceStepName;
|
||||
startedAtMs: number;
|
||||
durationMs: number;
|
||||
output: T;
|
||||
}
|
||||
|
||||
export interface ProcessTraceSnapshot {
|
||||
steps: {
|
||||
classify?: ProcessTraceStep<{
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: "regex" | "llm";
|
||||
}> | undefined;
|
||||
derive?: ProcessTraceStep<{
|
||||
phase: string;
|
||||
responseMode: string;
|
||||
shouldRespond: boolean;
|
||||
tier: 0 | 1 | 2;
|
||||
}> | undefined;
|
||||
plan?: ProcessTraceStep<{
|
||||
approach: string;
|
||||
mustAvoid: string[];
|
||||
mustInclude: string[];
|
||||
suggestedTools: string[];
|
||||
targetLengthChars: number;
|
||||
}> | undefined;
|
||||
toolCalls?: Array<
|
||||
ProcessTraceStep<{
|
||||
args: unknown;
|
||||
name: string;
|
||||
ok: boolean;
|
||||
resultSummary: string;
|
||||
}>
|
||||
> | undefined;
|
||||
respond?: ProcessTraceStep<{
|
||||
draftText: string;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
}> | undefined;
|
||||
critic?: ProcessTraceStep<{
|
||||
decision: "keep" | "rewrite";
|
||||
rationale: string;
|
||||
}> | undefined;
|
||||
validate?: ProcessTraceStep<{
|
||||
reasons: string[];
|
||||
status: "pass" | "reject" | "soft-warn";
|
||||
}> | undefined;
|
||||
};
|
||||
}
|
||||
|
||||
type ProcessTraceOutputByName = {
|
||||
classify: NonNullable<ProcessTraceSnapshot["steps"]["classify"]>["output"];
|
||||
critic: NonNullable<ProcessTraceSnapshot["steps"]["critic"]>["output"];
|
||||
derive: NonNullable<ProcessTraceSnapshot["steps"]["derive"]>["output"];
|
||||
plan: NonNullable<ProcessTraceSnapshot["steps"]["plan"]>["output"];
|
||||
respond: NonNullable<ProcessTraceSnapshot["steps"]["respond"]>["output"];
|
||||
"tool-call": NonNullable<
|
||||
NonNullable<ProcessTraceSnapshot["steps"]["toolCalls"]>[number]
|
||||
>["output"];
|
||||
validate: NonNullable<ProcessTraceSnapshot["steps"]["validate"]>["output"];
|
||||
};
|
||||
|
||||
const processTraceStorage = new AsyncLocalStorage<ProcessTraceCollector>();
|
||||
|
||||
export class ProcessTraceCollector {
|
||||
private readonly steps: ProcessTraceSnapshot["steps"] = {};
|
||||
|
||||
startStep<Name extends ProcessTraceStepName>(
|
||||
name: Name,
|
||||
): {
|
||||
finish: (output: ProcessTraceOutputByName[Name]) => void;
|
||||
} {
|
||||
const startedAtMs = Date.now();
|
||||
let finished = false;
|
||||
|
||||
return {
|
||||
finish: (output: ProcessTraceOutputByName[Name]): void => {
|
||||
if (finished) {
|
||||
return;
|
||||
}
|
||||
finished = true;
|
||||
|
||||
const step: ProcessTraceStep<ProcessTraceOutputByName[Name]> = {
|
||||
name,
|
||||
startedAtMs,
|
||||
durationMs: Math.max(0, Date.now() - startedAtMs),
|
||||
output,
|
||||
};
|
||||
|
||||
if (name === "tool-call") {
|
||||
this.steps.toolCalls = [
|
||||
...(this.steps.toolCalls ?? []),
|
||||
step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["tool-call"]
|
||||
>,
|
||||
];
|
||||
return;
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case "classify":
|
||||
this.steps.classify = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["classify"]
|
||||
>;
|
||||
return;
|
||||
case "critic":
|
||||
this.steps.critic = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["critic"]
|
||||
>;
|
||||
return;
|
||||
case "derive":
|
||||
this.steps.derive = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["derive"]
|
||||
>;
|
||||
return;
|
||||
case "plan":
|
||||
this.steps.plan = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["plan"]
|
||||
>;
|
||||
return;
|
||||
case "respond":
|
||||
this.steps.respond = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["respond"]
|
||||
>;
|
||||
return;
|
||||
case "validate":
|
||||
this.steps.validate = step as ProcessTraceStep<
|
||||
ProcessTraceOutputByName["validate"]
|
||||
>;
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): ProcessTraceSnapshot {
|
||||
return {
|
||||
steps: {
|
||||
...this.steps,
|
||||
toolCalls: this.steps.toolCalls
|
||||
? [...this.steps.toolCalls]
|
||||
: undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function runWithProcessTrace<T>(
|
||||
collector: ProcessTraceCollector,
|
||||
fn: () => T,
|
||||
): T {
|
||||
return processTraceStorage.run(collector, fn);
|
||||
}
|
||||
|
||||
export function getActiveProcessTrace(): ProcessTraceCollector | undefined {
|
||||
return processTraceStorage.getStore();
|
||||
}
|
||||
Reference in new issue
Block a user