Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent a900016be4
commit 221e9391e0
71 files changed
+7057 -102

No files matched your search

+379
View File
@@ -0,0 +1,379 @@
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import type { AgentRequest } from "@agent/types";
import type {
ClassifiedIntent,
DerivedState,
Intent,
OnboardingPhase,
} from "../state/onboarding-state.js";
import type { OnboardingSession } from "../types.js";
export type RoutingDecision =
| { kind: "abstain"; reason: string }
| { kind: "template-only"; templateKey: string }
| { kind: "tier0-only" }
| { kind: "tier1-direct"; skipPlanner: boolean }
| { kind: "tier1-with-tools" }
| { kind: "tier2-full" };
export const ROUTING_DECISION_KINDS = [
"abstain",
"template-only",
"tier0-only",
"tier1-direct",
"tier1-with-tools",
"tier2-full",
] as const;
export type RoutingDecisionKind = (typeof ROUTING_DECISION_KINDS)[number];
export interface RouterFeatureVector {
classifyConfidence: number;
classifySource: 0 | 1;
intentKind: string;
messageLength: number;
phaseKind: string;
responseModeTools: 0 | 1;
sessionCompleted: 0 | 1;
tier: 0 | 1 | 2;
trigger: string;
}
export interface LearnedRouterModel {
decisionKinds: RoutingDecisionKind[];
featureNames: string[];
trainedAt: string;
version: 1;
weights: number[][];
}
export interface RouterTrainingSample {
features: RouterFeatureVector;
label: RoutingDecisionKind;
}
export interface RouterInput {
classified: ClassifiedIntent;
derived: DerivedState;
request: AgentRequest;
session: OnboardingSession | null;
}
export const ROUTER_FEATURE_NAMES = [
"bias",
"classifyConfidence",
"classifySource",
"intentHash",
"phaseHash",
"triggerHash",
"messageLength",
"tier",
"responseModeTools",
"sessionCompleted",
] as const;
const modelCache = new Map<string, LearnedRouterModel | null>();
function round(value: number): number {
return Number(value.toFixed(6));
}
function clamp01(value: number): number {
return Math.max(0, Math.min(1, value));
}
function stableBucket(value: string): number {
const hash = createHash("sha256").update(value).digest();
return round((hash[0] ?? 0) / 255);
}
function dot(left: readonly number[], right: readonly number[]): number {
return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
}
function softmax(scores: readonly number[]): number[] {
const max = Math.max(...scores);
const exp = scores.map((score) => Math.exp(score - max));
const sum = exp.reduce((total, value) => total + value, 0);
return exp.map((value) => value / Math.max(1e-9, sum));
}
function decisionKindFromDecision(decision: RoutingDecision): RoutingDecisionKind {
return decision.kind;
}
function decisionFromKind(kind: RoutingDecisionKind, input: RouterInput): RoutingDecision {
switch (kind) {
case "abstain":
return {
kind: "abstain",
reason:
input.classified.intent.kind === "off-topic-to-other"
? "off-topic-strong"
: "learned-router",
};
case "template-only":
if (input.classified.intent.kind === "acknowledgement") {
return { kind, templateKey: "acknowledgement" };
}
if (
input.derived.phase.kind === "completed" ||
input.request.trigger === "completed"
) {
return { kind, templateKey: "completed" };
}
return { kind, templateKey: "greeting" };
case "tier1-direct":
return { kind, skipPlanner: false };
case "tier0-only":
case "tier1-with-tools":
case "tier2-full":
return { kind };
}
}
export function extractRouterFeatures(input: RouterInput): RouterFeatureVector {
return {
classifyConfidence: clamp01(input.classified.confidence),
classifySource: input.classified.source === "llm" ? 1 : 0,
intentKind: input.classified.intent.kind,
messageLength: input.request.userMessage?.trim().length ?? 0,
phaseKind: input.derived.phase.kind,
responseModeTools: input.derived.responseMode === "llm-with-tools" ? 1 : 0,
sessionCompleted: input.session?.status === "completed" ? 1 : 0,
tier: input.derived.tier,
trigger: input.request.trigger,
};
}
export function encodeRouterFeatures(features: RouterFeatureVector): number[] {
return [
1,
features.classifyConfidence,
features.classifySource,
stableBucket(features.intentKind),
stableBucket(features.phaseKind),
stableBucket(features.trigger),
Math.min(500, features.messageLength) / 500,
features.tier / 2,
features.responseModeTools,
features.sessionCompleted,
];
}
export function trainLearnedRouterModel(
samples: RouterTrainingSample[],
options?: {
epochs?: number | undefined;
learningRate?: number | undefined;
l2?: number | undefined;
trainedAt?: string | undefined;
},
): LearnedRouterModel {
const weights = ROUTING_DECISION_KINDS.map(() =>
ROUTER_FEATURE_NAMES.map(() => 0),
);
const epochs = options?.epochs ?? 200;
const learningRate = options?.learningRate ?? 0.2;
const l2 = options?.l2 ?? 0.001;
for (let epoch = 0; epoch < epochs; epoch++) {
for (const sample of samples) {
const encoded = encodeRouterFeatures(sample.features);
const targetIndex = ROUTING_DECISION_KINDS.indexOf(sample.label);
if (targetIndex < 0) {
continue;
}
const probs = softmax(weights.map((row) => dot(row, encoded)));
for (let decisionIndex = 0; decisionIndex < weights.length; decisionIndex++) {
const row = weights[decisionIndex];
if (!row) {
continue;
}
const error = (decisionIndex === targetIndex ? 1 : 0) - (probs[decisionIndex] ?? 0);
for (let featureIndex = 0; featureIndex < row.length; featureIndex++) {
row[featureIndex] =
(row[featureIndex] ?? 0) +
learningRate *
(error * (encoded[featureIndex] ?? 0) -
l2 * (row[featureIndex] ?? 0));
}
}
}
}
return {
decisionKinds: [...ROUTING_DECISION_KINDS],
featureNames: [...ROUTER_FEATURE_NAMES],
trainedAt: options?.trainedAt ?? new Date().toISOString(),
version: 1,
weights: weights.map((row) => row.map(round)),
};
}
export function predictRoutingDecisionKind(
model: LearnedRouterModel,
features: RouterFeatureVector,
): RoutingDecisionKind {
const encoded = encodeRouterFeatures(features);
const scores = model.weights.map((row) => dot(row, encoded));
const bestIndex = scores.reduce(
(best, score, index) => (score > (scores[best] ?? Number.NEGATIVE_INFINITY) ? index : best),
0,
);
return model.decisionKinds[bestIndex] ?? "tier0-only";
}
export function decideLearnedRouting(
input: RouterInput,
model: LearnedRouterModel,
): RoutingDecision {
return decisionFromKind(
predictRoutingDecisionKind(model, extractRouterFeatures(input)),
input,
);
}
export function loadLearnedRouterModel(path: string | undefined): LearnedRouterModel | null {
if (!path?.trim()) {
return null;
}
const cached = modelCache.get(path);
if (cached !== undefined) {
return cached;
}
if (!existsSync(path)) {
modelCache.set(path, null);
return null;
}
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as LearnedRouterModel;
if (
parsed.version !== 1 ||
parsed.featureNames.length !== ROUTER_FEATURE_NAMES.length ||
parsed.decisionKinds.length !== ROUTING_DECISION_KINDS.length ||
parsed.weights.length !== ROUTING_DECISION_KINDS.length ||
parsed.weights.some((row) => row.length !== ROUTER_FEATURE_NAMES.length)
) {
modelCache.set(path, null);
return null;
}
modelCache.set(path, parsed);
return parsed;
} catch {
modelCache.set(path, null);
return null;
}
}
export function saveLearnedRouterModel(path: string, model: LearnedRouterModel): void {
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, `${JSON.stringify(model, null, 2)}\n`);
modelCache.set(path, model);
}
function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
if (
phase.kind !== "fresh" &&
phase.kind !== "nickname-pending" &&
phase.kind !== "photo-pending" &&
phase.kind !== "both-pending"
) {
return false;
}
return (
intent.kind === "nickname-question" ||
intent.kind === "photo-question" ||
intent.kind === "help-how" ||
intent.kind === "help-where"
);
}
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
const { classified, derived, request, session } = input;
const { confidence, intent } = classified;
const { phase } = derived;
if (intent.kind === "off-topic-to-other" && confidence > 0.9) {
return {
kind: "abstain",
reason: "off-topic-strong",
};
}
if (intent.kind === "greeting" && confidence > 0.95) {
return {
kind: "template-only",
templateKey: "greeting",
};
}
if (intent.kind === "acknowledgement" && confidence > 0.9) {
return {
kind: "template-only",
templateKey: "acknowledgement",
};
}
if (phase.kind === "completed" && intent.kind !== "status-question") {
return {
kind: "template-only",
templateKey: "completed",
};
}
if (request.trigger === "completed" || session?.status === "completed") {
return {
kind: "template-only",
templateKey: "completed",
};
}
if (
phase.kind === "nickname-rejected" ||
intent.kind === "discord-verify-question" ||
needsStepOneHelp(phase, intent)
) {
return {
kind: "tier1-with-tools",
};
}
if (phase.kind === "deadline-near") {
return {
kind: "tier1-direct",
skipPlanner: false,
};
}
return {
kind: "tier0-only",
};
}
export function decideHybridRouting(
input: RouterInput,
options?: {
learnedModel?: LearnedRouterModel | null | undefined;
},
): RoutingDecision {
if (!options?.learnedModel) {
return decidePreGenerationRouting(input);
}
const staticDecision = decidePreGenerationRouting(input);
if (staticDecision.kind === "abstain" || staticDecision.kind === "template-only") {
return staticDecision;
}
return decideLearnedRouting(input, options.learnedModel);
}
export function routerDecisionKind(decision: RoutingDecision): RoutingDecisionKind {
return decisionKindFromDecision(decision);
}