Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent 6f09408423
commit 41dcfcdef7
71 files changed
+7057 -102

No files matched your search

+36 -4
View File
@@ -40,7 +40,14 @@ function average(values: number[]): number | null {
}
function sliceKeyToString(slice: SliceKey): string {
return `${slice.tier}:${slice.intent}:${slice.phase}`;
return [
slice.tier,
slice.intent,
slice.phase,
slice.routingDecision ?? "-",
slice.abstentionAction ?? "-",
slice.abstentionReason ?? "-",
].join(":");
}
function getSlice(entry: EvalArchiveEntry): SliceKey {
@@ -48,13 +55,38 @@ function getSlice(entry: EvalArchiveEntry): SliceKey {
tier: entry.tier,
intent: entry.derived.intent.kind,
phase: entry.derived.phase.kind,
...(entry.derived.routingDecision
? { routingDecision: entry.derived.routingDecision.kind }
: {}),
};
}
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null
? (value as Record<string, unknown>)
: {};
}
function hasAbstained(entry: EvalArchiveEntry): boolean {
const process = asRecord(entry.process);
const steps = asRecord(process["steps"]);
const classify = asRecord(steps["classify"]);
const output = asRecord(classify["output"]);
const abstention = asRecord(output["abstention"]);
const action = asRecord(abstention["action"]);
const kind = action["kind"];
return typeof kind === "string" && kind !== "proceed";
}
function getMetricValue(
entry: EvalArchiveEntry,
metricKey: "llmOverall" | "validatorPassRate",
metricKey: "abstentionRate" | "llmOverall" | "validatorPassRate",
): number | null {
if (metricKey === "abstentionRate") {
return hasAbstained(entry) ? 1 : 0;
}
if (metricKey === "validatorPassRate") {
return entry.validation.status === "pass" ? 1 : 0;
}
@@ -69,7 +101,7 @@ function getMetricValue(
export function computeDriftReportsFromEntries(
entries: EvalArchiveEntry[],
options?: {
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined;
nowMs?: number | undefined;
thresholdPct?: number | undefined;
},
@@ -136,7 +168,7 @@ export function computeDriftReportsFromEntries(
export async function computeDriftReports(
mongoConnection: MongoConnectionLike,
options?: {
metricKey?: "llmOverall" | "validatorPassRate" | undefined;
metricKey?: "abstentionRate" | "llmOverall" | "validatorPassRate" | undefined;
nowMs?: number | undefined;
},
): Promise<DriftReport[]> {