Add skill governance and runtime routing
This commit is contained in:
1 parent
a900016be4
commit
221e9391e0
71 files changed
+7057
-102
No files matched your search
@@ -0,0 +1,379 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
Intent,
|
||||
OnboardingPhase,
|
||||
} from "../state/onboarding-state.js";
|
||||
import type { OnboardingSession } from "../types.js";
|
||||
|
||||
export type RoutingDecision =
|
||||
| { kind: "abstain"; reason: string }
|
||||
| { kind: "template-only"; templateKey: string }
|
||||
| { kind: "tier0-only" }
|
||||
| { kind: "tier1-direct"; skipPlanner: boolean }
|
||||
| { kind: "tier1-with-tools" }
|
||||
| { kind: "tier2-full" };
|
||||
|
||||
export const ROUTING_DECISION_KINDS = [
|
||||
"abstain",
|
||||
"template-only",
|
||||
"tier0-only",
|
||||
"tier1-direct",
|
||||
"tier1-with-tools",
|
||||
"tier2-full",
|
||||
] as const;
|
||||
|
||||
export type RoutingDecisionKind = (typeof ROUTING_DECISION_KINDS)[number];
|
||||
|
||||
export interface RouterFeatureVector {
|
||||
classifyConfidence: number;
|
||||
classifySource: 0 | 1;
|
||||
intentKind: string;
|
||||
messageLength: number;
|
||||
phaseKind: string;
|
||||
responseModeTools: 0 | 1;
|
||||
sessionCompleted: 0 | 1;
|
||||
tier: 0 | 1 | 2;
|
||||
trigger: string;
|
||||
}
|
||||
|
||||
export interface LearnedRouterModel {
|
||||
decisionKinds: RoutingDecisionKind[];
|
||||
featureNames: string[];
|
||||
trainedAt: string;
|
||||
version: 1;
|
||||
weights: number[][];
|
||||
}
|
||||
|
||||
export interface RouterTrainingSample {
|
||||
features: RouterFeatureVector;
|
||||
label: RoutingDecisionKind;
|
||||
}
|
||||
|
||||
export interface RouterInput {
|
||||
classified: ClassifiedIntent;
|
||||
derived: DerivedState;
|
||||
request: AgentRequest;
|
||||
session: OnboardingSession | null;
|
||||
}
|
||||
|
||||
export const ROUTER_FEATURE_NAMES = [
|
||||
"bias",
|
||||
"classifyConfidence",
|
||||
"classifySource",
|
||||
"intentHash",
|
||||
"phaseHash",
|
||||
"triggerHash",
|
||||
"messageLength",
|
||||
"tier",
|
||||
"responseModeTools",
|
||||
"sessionCompleted",
|
||||
] as const;
|
||||
|
||||
const modelCache = new Map<string, LearnedRouterModel | null>();
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(6));
|
||||
}
|
||||
|
||||
function clamp01(value: number): number {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function stableBucket(value: string): number {
|
||||
const hash = createHash("sha256").update(value).digest();
|
||||
return round((hash[0] ?? 0) / 255);
|
||||
}
|
||||
|
||||
function dot(left: readonly number[], right: readonly number[]): number {
|
||||
return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
|
||||
}
|
||||
|
||||
function softmax(scores: readonly number[]): number[] {
|
||||
const max = Math.max(...scores);
|
||||
const exp = scores.map((score) => Math.exp(score - max));
|
||||
const sum = exp.reduce((total, value) => total + value, 0);
|
||||
return exp.map((value) => value / Math.max(1e-9, sum));
|
||||
}
|
||||
|
||||
function decisionKindFromDecision(decision: RoutingDecision): RoutingDecisionKind {
|
||||
return decision.kind;
|
||||
}
|
||||
|
||||
function decisionFromKind(kind: RoutingDecisionKind, input: RouterInput): RoutingDecision {
|
||||
switch (kind) {
|
||||
case "abstain":
|
||||
return {
|
||||
kind: "abstain",
|
||||
reason:
|
||||
input.classified.intent.kind === "off-topic-to-other"
|
||||
? "off-topic-strong"
|
||||
: "learned-router",
|
||||
};
|
||||
case "template-only":
|
||||
if (input.classified.intent.kind === "acknowledgement") {
|
||||
return { kind, templateKey: "acknowledgement" };
|
||||
}
|
||||
if (
|
||||
input.derived.phase.kind === "completed" ||
|
||||
input.request.trigger === "completed"
|
||||
) {
|
||||
return { kind, templateKey: "completed" };
|
||||
}
|
||||
return { kind, templateKey: "greeting" };
|
||||
case "tier1-direct":
|
||||
return { kind, skipPlanner: false };
|
||||
case "tier0-only":
|
||||
case "tier1-with-tools":
|
||||
case "tier2-full":
|
||||
return { kind };
|
||||
}
|
||||
}
|
||||
|
||||
export function extractRouterFeatures(input: RouterInput): RouterFeatureVector {
|
||||
return {
|
||||
classifyConfidence: clamp01(input.classified.confidence),
|
||||
classifySource: input.classified.source === "llm" ? 1 : 0,
|
||||
intentKind: input.classified.intent.kind,
|
||||
messageLength: input.request.userMessage?.trim().length ?? 0,
|
||||
phaseKind: input.derived.phase.kind,
|
||||
responseModeTools: input.derived.responseMode === "llm-with-tools" ? 1 : 0,
|
||||
sessionCompleted: input.session?.status === "completed" ? 1 : 0,
|
||||
tier: input.derived.tier,
|
||||
trigger: input.request.trigger,
|
||||
};
|
||||
}
|
||||
|
||||
export function encodeRouterFeatures(features: RouterFeatureVector): number[] {
|
||||
return [
|
||||
1,
|
||||
features.classifyConfidence,
|
||||
features.classifySource,
|
||||
stableBucket(features.intentKind),
|
||||
stableBucket(features.phaseKind),
|
||||
stableBucket(features.trigger),
|
||||
Math.min(500, features.messageLength) / 500,
|
||||
features.tier / 2,
|
||||
features.responseModeTools,
|
||||
features.sessionCompleted,
|
||||
];
|
||||
}
|
||||
|
||||
export function trainLearnedRouterModel(
|
||||
samples: RouterTrainingSample[],
|
||||
options?: {
|
||||
epochs?: number | undefined;
|
||||
learningRate?: number | undefined;
|
||||
l2?: number | undefined;
|
||||
trainedAt?: string | undefined;
|
||||
},
|
||||
): LearnedRouterModel {
|
||||
const weights = ROUTING_DECISION_KINDS.map(() =>
|
||||
ROUTER_FEATURE_NAMES.map(() => 0),
|
||||
);
|
||||
const epochs = options?.epochs ?? 200;
|
||||
const learningRate = options?.learningRate ?? 0.2;
|
||||
const l2 = options?.l2 ?? 0.001;
|
||||
|
||||
for (let epoch = 0; epoch < epochs; epoch++) {
|
||||
for (const sample of samples) {
|
||||
const encoded = encodeRouterFeatures(sample.features);
|
||||
const targetIndex = ROUTING_DECISION_KINDS.indexOf(sample.label);
|
||||
if (targetIndex < 0) {
|
||||
continue;
|
||||
}
|
||||
const probs = softmax(weights.map((row) => dot(row, encoded)));
|
||||
for (let decisionIndex = 0; decisionIndex < weights.length; decisionIndex++) {
|
||||
const row = weights[decisionIndex];
|
||||
if (!row) {
|
||||
continue;
|
||||
}
|
||||
const error = (decisionIndex === targetIndex ? 1 : 0) - (probs[decisionIndex] ?? 0);
|
||||
for (let featureIndex = 0; featureIndex < row.length; featureIndex++) {
|
||||
row[featureIndex] =
|
||||
(row[featureIndex] ?? 0) +
|
||||
learningRate *
|
||||
(error * (encoded[featureIndex] ?? 0) -
|
||||
l2 * (row[featureIndex] ?? 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
decisionKinds: [...ROUTING_DECISION_KINDS],
|
||||
featureNames: [...ROUTER_FEATURE_NAMES],
|
||||
trainedAt: options?.trainedAt ?? new Date().toISOString(),
|
||||
version: 1,
|
||||
weights: weights.map((row) => row.map(round)),
|
||||
};
|
||||
}
|
||||
|
||||
export function predictRoutingDecisionKind(
|
||||
model: LearnedRouterModel,
|
||||
features: RouterFeatureVector,
|
||||
): RoutingDecisionKind {
|
||||
const encoded = encodeRouterFeatures(features);
|
||||
const scores = model.weights.map((row) => dot(row, encoded));
|
||||
const bestIndex = scores.reduce(
|
||||
(best, score, index) => (score > (scores[best] ?? Number.NEGATIVE_INFINITY) ? index : best),
|
||||
0,
|
||||
);
|
||||
return model.decisionKinds[bestIndex] ?? "tier0-only";
|
||||
}
|
||||
|
||||
export function decideLearnedRouting(
|
||||
input: RouterInput,
|
||||
model: LearnedRouterModel,
|
||||
): RoutingDecision {
|
||||
return decisionFromKind(
|
||||
predictRoutingDecisionKind(model, extractRouterFeatures(input)),
|
||||
input,
|
||||
);
|
||||
}
|
||||
|
||||
export function loadLearnedRouterModel(path: string | undefined): LearnedRouterModel | null {
|
||||
if (!path?.trim()) {
|
||||
return null;
|
||||
}
|
||||
const cached = modelCache.get(path);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
if (!existsSync(path)) {
|
||||
modelCache.set(path, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(path, "utf8")) as LearnedRouterModel;
|
||||
if (
|
||||
parsed.version !== 1 ||
|
||||
parsed.featureNames.length !== ROUTER_FEATURE_NAMES.length ||
|
||||
parsed.decisionKinds.length !== ROUTING_DECISION_KINDS.length ||
|
||||
parsed.weights.length !== ROUTING_DECISION_KINDS.length ||
|
||||
parsed.weights.some((row) => row.length !== ROUTER_FEATURE_NAMES.length)
|
||||
) {
|
||||
modelCache.set(path, null);
|
||||
return null;
|
||||
}
|
||||
modelCache.set(path, parsed);
|
||||
return parsed;
|
||||
} catch {
|
||||
modelCache.set(path, null);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function saveLearnedRouterModel(path: string, model: LearnedRouterModel): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, `${JSON.stringify(model, null, 2)}\n`);
|
||||
modelCache.set(path, model);
|
||||
}
|
||||
|
||||
function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
|
||||
if (
|
||||
phase.kind !== "fresh" &&
|
||||
phase.kind !== "nickname-pending" &&
|
||||
phase.kind !== "photo-pending" &&
|
||||
phase.kind !== "both-pending"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
intent.kind === "nickname-question" ||
|
||||
intent.kind === "photo-question" ||
|
||||
intent.kind === "help-how" ||
|
||||
intent.kind === "help-where"
|
||||
);
|
||||
}
|
||||
|
||||
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
|
||||
const { classified, derived, request, session } = input;
|
||||
const { confidence, intent } = classified;
|
||||
const { phase } = derived;
|
||||
|
||||
if (intent.kind === "off-topic-to-other" && confidence > 0.9) {
|
||||
return {
|
||||
kind: "abstain",
|
||||
reason: "off-topic-strong",
|
||||
};
|
||||
}
|
||||
|
||||
if (intent.kind === "greeting" && confidence > 0.95) {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "greeting",
|
||||
};
|
||||
}
|
||||
|
||||
if (intent.kind === "acknowledgement" && confidence > 0.9) {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "acknowledgement",
|
||||
};
|
||||
}
|
||||
|
||||
if (phase.kind === "completed" && intent.kind !== "status-question") {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "completed",
|
||||
};
|
||||
}
|
||||
|
||||
if (request.trigger === "completed" || session?.status === "completed") {
|
||||
return {
|
||||
kind: "template-only",
|
||||
templateKey: "completed",
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
phase.kind === "nickname-rejected" ||
|
||||
intent.kind === "discord-verify-question" ||
|
||||
needsStepOneHelp(phase, intent)
|
||||
) {
|
||||
return {
|
||||
kind: "tier1-with-tools",
|
||||
};
|
||||
}
|
||||
|
||||
if (phase.kind === "deadline-near") {
|
||||
return {
|
||||
kind: "tier1-direct",
|
||||
skipPlanner: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "tier0-only",
|
||||
};
|
||||
}
|
||||
|
||||
export function decideHybridRouting(
|
||||
input: RouterInput,
|
||||
options?: {
|
||||
learnedModel?: LearnedRouterModel | null | undefined;
|
||||
},
|
||||
): RoutingDecision {
|
||||
if (!options?.learnedModel) {
|
||||
return decidePreGenerationRouting(input);
|
||||
}
|
||||
|
||||
const staticDecision = decidePreGenerationRouting(input);
|
||||
if (staticDecision.kind === "abstain" || staticDecision.kind === "template-only") {
|
||||
return staticDecision;
|
||||
}
|
||||
|
||||
return decideLearnedRouting(input, options.learnedModel);
|
||||
}
|
||||
|
||||
export function routerDecisionKind(decision: RoutingDecision): RoutingDecisionKind {
|
||||
return decisionKindFromDecision(decision);
|
||||
}
|
||||
Reference in new issue
Block a user