Add skill governance and runtime routing
This commit is contained in:
1 parent
a900016be4
commit
221e9391e0
71 files changed
+7057
-102
No files matched your search
+36
-4
@@ -40,7 +40,14 @@ function average(values: number[]): number | null {
|
||||
}
|
||||
|
||||
function sliceKeyToString(slice: SliceKey): string {
|
||||
return `${slice.tier}:${slice.intent}:${slice.phase}`;
|
||||
return [
|
||||
slice.tier,
|
||||
slice.intent,
|
||||
slice.phase,
|
||||
slice.routingDecision ?? "-",
|
||||
slice.abstentionAction ?? "-",
|
||||
slice.abstentionReason ?? "-",
|
||||
].join(":");
|
||||
}
|
||||
|
||||
function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
@@ -48,13 +55,38 @@ function getSlice(entry: EvalArchiveEntry): SliceKey {
|
||||
tier: entry.tier,
|
||||
intent: entry.derived.intent.kind,
|
||||
phase: entry.derived.phase.kind,
|
||||
...(entry.derived.routingDecision
|
||||
? { routingDecision: entry.derived.routingDecision.kind }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function hasAbstained(entry: EvalArchiveEntry): boolean {
|
||||
const process = asRecord(entry.process);
|
||||
const steps = asRecord(process["steps"]);
|
||||
const classify = asRecord(steps["classify"]);
|
||||
const output = asRecord(classify["output"]);
|
||||
const abstention = asRecord(output["abstention"]);
|
||||
const action = asRecord(abstention["action"]);
|
||||
const kind = action["kind"];
|
||||
|
||||
return typeof kind === "string" && kind !== "proceed";
|
||||
}
|
||||
|
||||
function getMetricValue(
|
||||
entry: EvalArchiveEntry,
|
||||
metricKey: "llmOverall" | "validatorPassRate",
|
||||
metricKey: "abstentionRate" | "llmOverall" | "validatorPassRate",
|
||||
): number | null {
|
||||
if (metricKey === "abstentionRate") {
|
||||
return hasAbstained(entry) ? 1 : 0;
|
||||
}
|
||||
|
||||
if (metricKey === "validatorPassRate") {
|
||||
return entry.validation.status === "pass" ? 1 : 0;
|
||||
}
|
||||
@@ -69,7 +101,7 @@ function getMetricValue(
|
||||
export function computeDriftReportsFromEntries(
|
||||
entries: EvalArchiveEntry[],
|
||||
options?: {
|
||||
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
|
||||
metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined;
|
||||
nowMs?: number | undefined;
|
||||
thresholdPct?: number | undefined;
|
||||
},
|
||||
@@ -136,7 +168,7 @@ export function computeDriftReportsFromEntries(
|
||||
export async function computeDriftReports(
|
||||
mongoConnection: MongoConnectionLike,
|
||||
options?: {
|
||||
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
|
||||
metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined;
|
||||
nowMs?: number | undefined;
|
||||
},
|
||||
): Promise<DriftReport[]> {
|
||||
|
||||
Reference in new issue
Block a user