Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent 6f09408423
commit 41dcfcdef7
71 files changed
+7057 -102

No files matched your search

+100 -4
View File
@@ -12,9 +12,13 @@ export interface SliceKey {
tier: 0 | 1 | 2;
intent: string;
phase: string;
routingDecision?: string | undefined;
abstentionAction?: string | undefined;
abstentionReason?: string | undefined;
}
export interface SliceMetrics {
abstentionCount: number;
count: number;
llmOverallSum: number;
llmOverallCount: number;
@@ -29,6 +33,7 @@ export interface SliceMetrics {
export interface SliceReportRow {
slice: SliceKey;
abstentionRate: number;
count: number;
llmOverallAvg: number | null;
processJudgeAvg: number | null;
@@ -50,6 +55,7 @@ type AggregatorGarnetClient = Pick<
>;
interface EvalResultObservation {
abstentionAction: string | null;
latencyMs: number | null;
llmOverall: number | null;
processJudgeScore: number | null;
@@ -133,6 +139,39 @@ function extractValidatorStatus(message: Record<string, string>): string | null
return typeof status === "string" && status.trim() ? status : null;
}
function extractAbstention(
message: Record<string, string>,
): { action: string | null; reason: string | null } {
const directAction = message["abstentionAction"]?.trim();
const directReason = message["abstentionReason"]?.trim();
if (directAction) {
return {
action: directAction,
reason: directReason || null,
};
}
const process = parseJsonRecord(message["process"]);
const steps = asRecord(process["steps"]);
const classify = asRecord(steps["classify"]);
const output = asRecord(classify["output"]);
const abstention = asRecord(output["abstention"]);
const action = asRecord(abstention["action"]);
const kind = action["kind"];
const reason = abstention["reason"];
return {
action: typeof kind === "string" && kind.trim() ? kind : null,
reason: typeof reason === "string" && reason.trim() ? reason : null,
};
}
function extractRoutingDecision(derived: Record<string, unknown>): string | null {
const routingDecision = asRecord(derived["routingDecision"]);
const kind = routingDecision["kind"];
return typeof kind === "string" && kind.trim() ? kind : null;
}
function parseObservation(entry: StreamEntry): EvalResultObservation | null {
const derived = parseJsonRecord(entry.message["derived"]);
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
@@ -146,11 +185,22 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
return null;
}
const abstention = extractAbstention(entry.message);
const routingDecision = extractRoutingDecision(derived);
return {
abstentionAction: abstention.action,
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
llmOverall: extractLlmOverall(entry.message),
processJudgeScore: extractProcessJudgeScore(entry.message),
slice: { tier, intent, phase },
slice: {
tier,
intent,
phase,
...(routingDecision ? { routingDecision } : {}),
...(abstention.action ? { abstentionAction: abstention.action } : {}),
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
},
validatorStatus: extractValidatorStatus(entry.message),
};
}
@@ -185,6 +235,7 @@ function parseLatencySamples(hash: Record<string, string>): number[] {
function metricsFromHash(hash: Record<string, string>): SliceMetrics {
return {
abstentionCount: parseNumberField(hash, "abstentionCount"),
count: parseNumberField(hash, "count"),
llmOverallSum: parseNumberField(hash, "llmOverallSum"),
llmOverallCount: parseNumberField(hash, "llmOverallCount"),
@@ -200,6 +251,7 @@ function metricsFromHash(hash: Record<string, string>): SliceMetrics {
function metricsToHash(metrics: SliceMetrics): Record<string, string> {
return {
abstentionCount: String(metrics.abstentionCount),
count: String(metrics.count),
llmOverallSum: String(metrics.llmOverallSum),
llmOverallCount: String(metrics.llmOverallCount),
@@ -229,6 +281,12 @@ function updateMetrics(
}
return {
abstentionCount:
current.abstentionCount +
(observation.abstentionAction &&
observation.abstentionAction !== "proceed"
? 1
: 0),
count: current.count + 1,
llmOverallSum:
current.llmOverallSum + (observation.llmOverall ?? 0),
@@ -282,13 +340,31 @@ function average(sum: number, count: number): number | null {
}
function sliceKeyFromString(value: string): SliceKey | null {
const [tierRaw, intent, phase] = value.split(":");
const parts = value.split(":");
const [tierRaw, intent, phase] = parts;
const tier = parseTier(tierRaw);
if (tier === null || !intent || !phase) {
return null;
}
return { tier, intent, phase };
const routingDecision = parts.length >= 6 ? parts[3] : undefined;
const abstentionAction = parts.length >= 6 ? parts[4] : parts[3];
const abstentionReason = parts.length >= 6 ? parts[5] : parts[4];
return {
tier,
intent,
phase,
...(routingDecision && routingDecision !== "-"
? { routingDecision }
: {}),
...(abstentionAction && abstentionAction !== "-"
? { abstentionAction }
: {}),
...(abstentionReason && abstentionReason !== "-"
? { abstentionReason }
: {}),
};
}
function sliceKeyFromRedisKey(key: string): SliceKey | null {
@@ -300,7 +376,25 @@ function sliceKeyFromRedisKey(key: string): SliceKey | null {
}
export function sliceKeyToString(key: SliceKey): string {
return `${key.tier}:${key.intent}:${key.phase}`;
const base = `${key.tier}:${key.intent}:${key.phase}`;
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
return base;
}
if (key.routingDecision) {
return [
base,
key.routingDecision,
key.abstentionAction ?? "-",
key.abstentionReason ?? "-",
].join(":");
}
return [
base,
key.abstentionAction ?? "-",
key.abstentionReason ?? "-",
].join(":");
}
export function getSliceAggregateKey(key: SliceKey): string {
@@ -313,6 +407,8 @@ export function sliceMetricsToReportRow(
): SliceReportRow {
return {
slice,
abstentionRate:
metrics.count === 0 ? 0 : round(metrics.abstentionCount / metrics.count),
count: metrics.count,
llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount),
processJudgeAvg: average(