Add skill governance and runtime routing
This commit is contained in:
1 parent
6f09408423
commit
41dcfcdef7
71 files changed
+7057
-102
No files matched your search
+100
-4
@@ -12,9 +12,13 @@ export interface SliceKey {
|
||||
tier: 0 | 1 | 2;
|
||||
intent: string;
|
||||
phase: string;
|
||||
routingDecision?: string | undefined;
|
||||
abstentionAction?: string | undefined;
|
||||
abstentionReason?: string | undefined;
|
||||
}
|
||||
|
||||
export interface SliceMetrics {
|
||||
abstentionCount: number;
|
||||
count: number;
|
||||
llmOverallSum: number;
|
||||
llmOverallCount: number;
|
||||
@@ -29,6 +33,7 @@ export interface SliceMetrics {
|
||||
|
||||
export interface SliceReportRow {
|
||||
slice: SliceKey;
|
||||
abstentionRate: number;
|
||||
count: number;
|
||||
llmOverallAvg: number | null;
|
||||
processJudgeAvg: number | null;
|
||||
@@ -50,6 +55,7 @@ type AggregatorGarnetClient = Pick<
|
||||
>;
|
||||
|
||||
interface EvalResultObservation {
|
||||
abstentionAction: string | null;
|
||||
latencyMs: number | null;
|
||||
llmOverall: number | null;
|
||||
processJudgeScore: number | null;
|
||||
@@ -133,6 +139,39 @@ function extractValidatorStatus(message: Record<string, string>): string | null
|
||||
return typeof status === "string" && status.trim() ? status : null;
|
||||
}
|
||||
|
||||
function extractAbstention(
|
||||
message: Record<string, string>,
|
||||
): { action: string | null; reason: string | null } {
|
||||
const directAction = message["abstentionAction"]?.trim();
|
||||
const directReason = message["abstentionReason"]?.trim();
|
||||
if (directAction) {
|
||||
return {
|
||||
action: directAction,
|
||||
reason: directReason || null,
|
||||
};
|
||||
}
|
||||
|
||||
const process = parseJsonRecord(message["process"]);
|
||||
const steps = asRecord(process["steps"]);
|
||||
const classify = asRecord(steps["classify"]);
|
||||
const output = asRecord(classify["output"]);
|
||||
const abstention = asRecord(output["abstention"]);
|
||||
const action = asRecord(abstention["action"]);
|
||||
const kind = action["kind"];
|
||||
const reason = abstention["reason"];
|
||||
|
||||
return {
|
||||
action: typeof kind === "string" && kind.trim() ? kind : null,
|
||||
reason: typeof reason === "string" && reason.trim() ? reason : null,
|
||||
};
|
||||
}
|
||||
|
||||
function extractRoutingDecision(derived: Record<string, unknown>): string | null {
|
||||
const routingDecision = asRecord(derived["routingDecision"]);
|
||||
const kind = routingDecision["kind"];
|
||||
return typeof kind === "string" && kind.trim() ? kind : null;
|
||||
}
|
||||
|
||||
function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
||||
const derived = parseJsonRecord(entry.message["derived"]);
|
||||
const tier = parseTier(entry.message["tier"]) ?? parseTier(derived["tier"]);
|
||||
@@ -146,11 +185,22 @@ function parseObservation(entry: StreamEntry): EvalResultObservation | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
const abstention = extractAbstention(entry.message);
|
||||
const routingDecision = extractRoutingDecision(derived);
|
||||
|
||||
return {
|
||||
abstentionAction: abstention.action,
|
||||
latencyMs: parseFiniteNumber(entry.message["latencyMs"]),
|
||||
llmOverall: extractLlmOverall(entry.message),
|
||||
processJudgeScore: extractProcessJudgeScore(entry.message),
|
||||
slice: { tier, intent, phase },
|
||||
slice: {
|
||||
tier,
|
||||
intent,
|
||||
phase,
|
||||
...(routingDecision ? { routingDecision } : {}),
|
||||
...(abstention.action ? { abstentionAction: abstention.action } : {}),
|
||||
...(abstention.reason ? { abstentionReason: abstention.reason } : {}),
|
||||
},
|
||||
validatorStatus: extractValidatorStatus(entry.message),
|
||||
};
|
||||
}
|
||||
@@ -185,6 +235,7 @@ function parseLatencySamples(hash: Record<string, string>): number[] {
|
||||
|
||||
function metricsFromHash(hash: Record<string, string>): SliceMetrics {
|
||||
return {
|
||||
abstentionCount: parseNumberField(hash, "abstentionCount"),
|
||||
count: parseNumberField(hash, "count"),
|
||||
llmOverallSum: parseNumberField(hash, "llmOverallSum"),
|
||||
llmOverallCount: parseNumberField(hash, "llmOverallCount"),
|
||||
@@ -200,6 +251,7 @@ function metricsFromHash(hash: Record<string, string>): SliceMetrics {
|
||||
|
||||
function metricsToHash(metrics: SliceMetrics): Record<string, string> {
|
||||
return {
|
||||
abstentionCount: String(metrics.abstentionCount),
|
||||
count: String(metrics.count),
|
||||
llmOverallSum: String(metrics.llmOverallSum),
|
||||
llmOverallCount: String(metrics.llmOverallCount),
|
||||
@@ -229,6 +281,12 @@ function updateMetrics(
|
||||
}
|
||||
|
||||
return {
|
||||
abstentionCount:
|
||||
current.abstentionCount +
|
||||
(observation.abstentionAction &&
|
||||
observation.abstentionAction !== "proceed"
|
||||
? 1
|
||||
: 0),
|
||||
count: current.count + 1,
|
||||
llmOverallSum:
|
||||
current.llmOverallSum + (observation.llmOverall ?? 0),
|
||||
@@ -282,13 +340,31 @@ function average(sum: number, count: number): number | null {
|
||||
}
|
||||
|
||||
function sliceKeyFromString(value: string): SliceKey | null {
|
||||
const [tierRaw, intent, phase] = value.split(":");
|
||||
const parts = value.split(":");
|
||||
const [tierRaw, intent, phase] = parts;
|
||||
const tier = parseTier(tierRaw);
|
||||
if (tier === null || !intent || !phase) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { tier, intent, phase };
|
||||
const routingDecision = parts.length >= 6 ? parts[3] : undefined;
|
||||
const abstentionAction = parts.length >= 6 ? parts[4] : parts[3];
|
||||
const abstentionReason = parts.length >= 6 ? parts[5] : parts[4];
|
||||
|
||||
return {
|
||||
tier,
|
||||
intent,
|
||||
phase,
|
||||
...(routingDecision && routingDecision !== "-"
|
||||
? { routingDecision }
|
||||
: {}),
|
||||
...(abstentionAction && abstentionAction !== "-"
|
||||
? { abstentionAction }
|
||||
: {}),
|
||||
...(abstentionReason && abstentionReason !== "-"
|
||||
? { abstentionReason }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function sliceKeyFromRedisKey(key: string): SliceKey | null {
|
||||
@@ -300,7 +376,25 @@ function sliceKeyFromRedisKey(key: string): SliceKey | null {
|
||||
}
|
||||
|
||||
export function sliceKeyToString(key: SliceKey): string {
|
||||
return `${key.tier}:${key.intent}:${key.phase}`;
|
||||
const base = `${key.tier}:${key.intent}:${key.phase}`;
|
||||
if (!key.routingDecision && !key.abstentionAction && !key.abstentionReason) {
|
||||
return base;
|
||||
}
|
||||
|
||||
if (key.routingDecision) {
|
||||
return [
|
||||
base,
|
||||
key.routingDecision,
|
||||
key.abstentionAction ?? "-",
|
||||
key.abstentionReason ?? "-",
|
||||
].join(":");
|
||||
}
|
||||
|
||||
return [
|
||||
base,
|
||||
key.abstentionAction ?? "-",
|
||||
key.abstentionReason ?? "-",
|
||||
].join(":");
|
||||
}
|
||||
|
||||
export function getSliceAggregateKey(key: SliceKey): string {
|
||||
@@ -313,6 +407,8 @@ export function sliceMetricsToReportRow(
|
||||
): SliceReportRow {
|
||||
return {
|
||||
slice,
|
||||
abstentionRate:
|
||||
metrics.count === 0 ? 0 : round(metrics.abstentionCount / metrics.count),
|
||||
count: metrics.count,
|
||||
llmOverallAvg: average(metrics.llmOverallSum, metrics.llmOverallCount),
|
||||
processJudgeAvg: average(
|
||||
|
||||
Reference in new issue
Block a user