380 lines
10 KiB
TypeScript
380 lines
10 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { dirname } from "node:path";
|
|
|
|
import type { AgentRequest } from "@agent/types";
|
|
import type {
|
|
ClassifiedIntent,
|
|
DerivedState,
|
|
Intent,
|
|
OnboardingPhase,
|
|
} from "../state/onboarding-state.js";
|
|
import type { OnboardingSession } from "../types.js";
|
|
|
|
export type RoutingDecision =
|
|
| { kind: "abstain"; reason: string }
|
|
| { kind: "template-only"; templateKey: string }
|
|
| { kind: "tier0-only" }
|
|
| { kind: "tier1-direct"; skipPlanner: boolean }
|
|
| { kind: "tier1-with-tools" }
|
|
| { kind: "tier2-full" };
|
|
|
|
export const ROUTING_DECISION_KINDS = [
|
|
"abstain",
|
|
"template-only",
|
|
"tier0-only",
|
|
"tier1-direct",
|
|
"tier1-with-tools",
|
|
"tier2-full",
|
|
] as const;
|
|
|
|
export type RoutingDecisionKind = (typeof ROUTING_DECISION_KINDS)[number];
|
|
|
|
export interface RouterFeatureVector {
|
|
classifyConfidence: number;
|
|
classifySource: 0 | 1;
|
|
intentKind: string;
|
|
messageLength: number;
|
|
phaseKind: string;
|
|
responseModeTools: 0 | 1;
|
|
sessionCompleted: 0 | 1;
|
|
tier: 0 | 1 | 2;
|
|
trigger: string;
|
|
}
|
|
|
|
export interface LearnedRouterModel {
|
|
decisionKinds: RoutingDecisionKind[];
|
|
featureNames: string[];
|
|
trainedAt: string;
|
|
version: 1;
|
|
weights: number[][];
|
|
}
|
|
|
|
export interface RouterTrainingSample {
|
|
features: RouterFeatureVector;
|
|
label: RoutingDecisionKind;
|
|
}
|
|
|
|
export interface RouterInput {
|
|
classified: ClassifiedIntent;
|
|
derived: DerivedState;
|
|
request: AgentRequest;
|
|
session: OnboardingSession | null;
|
|
}
|
|
|
|
export const ROUTER_FEATURE_NAMES = [
|
|
"bias",
|
|
"classifyConfidence",
|
|
"classifySource",
|
|
"intentHash",
|
|
"phaseHash",
|
|
"triggerHash",
|
|
"messageLength",
|
|
"tier",
|
|
"responseModeTools",
|
|
"sessionCompleted",
|
|
] as const;
|
|
|
|
const modelCache = new Map<string, LearnedRouterModel | null>();
|
|
|
|
function round(value: number): number {
|
|
return Number(value.toFixed(6));
|
|
}
|
|
|
|
function clamp01(value: number): number {
|
|
return Math.max(0, Math.min(1, value));
|
|
}
|
|
|
|
function stableBucket(value: string): number {
|
|
const hash = createHash("sha256").update(value).digest();
|
|
return round((hash[0] ?? 0) / 255);
|
|
}
|
|
|
|
function dot(left: readonly number[], right: readonly number[]): number {
|
|
return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
|
|
}
|
|
|
|
function softmax(scores: readonly number[]): number[] {
|
|
const max = Math.max(...scores);
|
|
const exp = scores.map((score) => Math.exp(score - max));
|
|
const sum = exp.reduce((total, value) => total + value, 0);
|
|
return exp.map((value) => value / Math.max(1e-9, sum));
|
|
}
|
|
|
|
function decisionKindFromDecision(decision: RoutingDecision): RoutingDecisionKind {
|
|
return decision.kind;
|
|
}
|
|
|
|
function decisionFromKind(kind: RoutingDecisionKind, input: RouterInput): RoutingDecision {
|
|
switch (kind) {
|
|
case "abstain":
|
|
return {
|
|
kind: "abstain",
|
|
reason:
|
|
input.classified.intent.kind === "off-topic-to-other"
|
|
? "off-topic-strong"
|
|
: "learned-router",
|
|
};
|
|
case "template-only":
|
|
if (input.classified.intent.kind === "acknowledgement") {
|
|
return { kind, templateKey: "acknowledgement" };
|
|
}
|
|
if (
|
|
input.derived.phase.kind === "completed" ||
|
|
input.request.trigger === "completed"
|
|
) {
|
|
return { kind, templateKey: "completed" };
|
|
}
|
|
return { kind, templateKey: "greeting" };
|
|
case "tier1-direct":
|
|
return { kind, skipPlanner: false };
|
|
case "tier0-only":
|
|
case "tier1-with-tools":
|
|
case "tier2-full":
|
|
return { kind };
|
|
}
|
|
}
|
|
|
|
export function extractRouterFeatures(input: RouterInput): RouterFeatureVector {
|
|
return {
|
|
classifyConfidence: clamp01(input.classified.confidence),
|
|
classifySource: input.classified.source === "llm" ? 1 : 0,
|
|
intentKind: input.classified.intent.kind,
|
|
messageLength: input.request.userMessage?.trim().length ?? 0,
|
|
phaseKind: input.derived.phase.kind,
|
|
responseModeTools: input.derived.responseMode === "llm-with-tools" ? 1 : 0,
|
|
sessionCompleted: input.session?.status === "completed" ? 1 : 0,
|
|
tier: input.derived.tier,
|
|
trigger: input.request.trigger,
|
|
};
|
|
}
|
|
|
|
export function encodeRouterFeatures(features: RouterFeatureVector): number[] {
|
|
return [
|
|
1,
|
|
features.classifyConfidence,
|
|
features.classifySource,
|
|
stableBucket(features.intentKind),
|
|
stableBucket(features.phaseKind),
|
|
stableBucket(features.trigger),
|
|
Math.min(500, features.messageLength) / 500,
|
|
features.tier / 2,
|
|
features.responseModeTools,
|
|
features.sessionCompleted,
|
|
];
|
|
}
|
|
|
|
export function trainLearnedRouterModel(
|
|
samples: RouterTrainingSample[],
|
|
options?: {
|
|
epochs?: number | undefined;
|
|
learningRate?: number | undefined;
|
|
l2?: number | undefined;
|
|
trainedAt?: string | undefined;
|
|
},
|
|
): LearnedRouterModel {
|
|
const weights = ROUTING_DECISION_KINDS.map(() =>
|
|
ROUTER_FEATURE_NAMES.map(() => 0),
|
|
);
|
|
const epochs = options?.epochs ?? 200;
|
|
const learningRate = options?.learningRate ?? 0.2;
|
|
const l2 = options?.l2 ?? 0.001;
|
|
|
|
for (let epoch = 0; epoch < epochs; epoch++) {
|
|
for (const sample of samples) {
|
|
const encoded = encodeRouterFeatures(sample.features);
|
|
const targetIndex = ROUTING_DECISION_KINDS.indexOf(sample.label);
|
|
if (targetIndex < 0) {
|
|
continue;
|
|
}
|
|
const probs = softmax(weights.map((row) => dot(row, encoded)));
|
|
for (let decisionIndex = 0; decisionIndex < weights.length; decisionIndex++) {
|
|
const row = weights[decisionIndex];
|
|
if (!row) {
|
|
continue;
|
|
}
|
|
const error = (decisionIndex === targetIndex ? 1 : 0) - (probs[decisionIndex] ?? 0);
|
|
for (let featureIndex = 0; featureIndex < row.length; featureIndex++) {
|
|
row[featureIndex] =
|
|
(row[featureIndex] ?? 0) +
|
|
learningRate *
|
|
(error * (encoded[featureIndex] ?? 0) -
|
|
l2 * (row[featureIndex] ?? 0));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
decisionKinds: [...ROUTING_DECISION_KINDS],
|
|
featureNames: [...ROUTER_FEATURE_NAMES],
|
|
trainedAt: options?.trainedAt ?? new Date().toISOString(),
|
|
version: 1,
|
|
weights: weights.map((row) => row.map(round)),
|
|
};
|
|
}
|
|
|
|
export function predictRoutingDecisionKind(
|
|
model: LearnedRouterModel,
|
|
features: RouterFeatureVector,
|
|
): RoutingDecisionKind {
|
|
const encoded = encodeRouterFeatures(features);
|
|
const scores = model.weights.map((row) => dot(row, encoded));
|
|
const bestIndex = scores.reduce(
|
|
(best, score, index) => (score > (scores[best] ?? Number.NEGATIVE_INFINITY) ? index : best),
|
|
0,
|
|
);
|
|
return model.decisionKinds[bestIndex] ?? "tier0-only";
|
|
}
|
|
|
|
export function decideLearnedRouting(
|
|
input: RouterInput,
|
|
model: LearnedRouterModel,
|
|
): RoutingDecision {
|
|
return decisionFromKind(
|
|
predictRoutingDecisionKind(model, extractRouterFeatures(input)),
|
|
input,
|
|
);
|
|
}
|
|
|
|
export function loadLearnedRouterModel(path: string | undefined): LearnedRouterModel | null {
|
|
if (!path?.trim()) {
|
|
return null;
|
|
}
|
|
const cached = modelCache.get(path);
|
|
if (cached !== undefined) {
|
|
return cached;
|
|
}
|
|
if (!existsSync(path)) {
|
|
modelCache.set(path, null);
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const parsed = JSON.parse(readFileSync(path, "utf8")) as LearnedRouterModel;
|
|
if (
|
|
parsed.version !== 1 ||
|
|
parsed.featureNames.length !== ROUTER_FEATURE_NAMES.length ||
|
|
parsed.decisionKinds.length !== ROUTING_DECISION_KINDS.length ||
|
|
parsed.weights.length !== ROUTING_DECISION_KINDS.length ||
|
|
parsed.weights.some((row) => row.length !== ROUTER_FEATURE_NAMES.length)
|
|
) {
|
|
modelCache.set(path, null);
|
|
return null;
|
|
}
|
|
modelCache.set(path, parsed);
|
|
return parsed;
|
|
} catch {
|
|
modelCache.set(path, null);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function saveLearnedRouterModel(path: string, model: LearnedRouterModel): void {
|
|
mkdirSync(dirname(path), { recursive: true });
|
|
writeFileSync(path, `${JSON.stringify(model, null, 2)}\n`);
|
|
modelCache.set(path, model);
|
|
}
|
|
|
|
function needsStepOneHelp(phase: OnboardingPhase, intent: Intent): boolean {
|
|
if (
|
|
phase.kind !== "fresh" &&
|
|
phase.kind !== "nickname-pending" &&
|
|
phase.kind !== "photo-pending" &&
|
|
phase.kind !== "both-pending"
|
|
) {
|
|
return false;
|
|
}
|
|
|
|
return (
|
|
intent.kind === "nickname-question" ||
|
|
intent.kind === "photo-question" ||
|
|
intent.kind === "help-how" ||
|
|
intent.kind === "help-where"
|
|
);
|
|
}
|
|
|
|
export function decidePreGenerationRouting(input: RouterInput): RoutingDecision {
|
|
const { classified, derived, request, session } = input;
|
|
const { confidence, intent } = classified;
|
|
const { phase } = derived;
|
|
|
|
if (intent.kind === "off-topic-to-other" && confidence > 0.9) {
|
|
return {
|
|
kind: "abstain",
|
|
reason: "off-topic-strong",
|
|
};
|
|
}
|
|
|
|
if (intent.kind === "greeting" && confidence > 0.95) {
|
|
return {
|
|
kind: "template-only",
|
|
templateKey: "greeting",
|
|
};
|
|
}
|
|
|
|
if (intent.kind === "acknowledgement" && confidence > 0.9) {
|
|
return {
|
|
kind: "template-only",
|
|
templateKey: "acknowledgement",
|
|
};
|
|
}
|
|
|
|
if (phase.kind === "completed" && intent.kind !== "status-question") {
|
|
return {
|
|
kind: "template-only",
|
|
templateKey: "completed",
|
|
};
|
|
}
|
|
|
|
if (request.trigger === "completed" || session?.status === "completed") {
|
|
return {
|
|
kind: "template-only",
|
|
templateKey: "completed",
|
|
};
|
|
}
|
|
|
|
if (
|
|
phase.kind === "nickname-rejected" ||
|
|
intent.kind === "discord-verify-question" ||
|
|
needsStepOneHelp(phase, intent)
|
|
) {
|
|
return {
|
|
kind: "tier1-with-tools",
|
|
};
|
|
}
|
|
|
|
if (phase.kind === "deadline-near") {
|
|
return {
|
|
kind: "tier1-direct",
|
|
skipPlanner: false,
|
|
};
|
|
}
|
|
|
|
return {
|
|
kind: "tier0-only",
|
|
};
|
|
}
|
|
|
|
export function decideHybridRouting(
|
|
input: RouterInput,
|
|
options?: {
|
|
learnedModel?: LearnedRouterModel | null | undefined;
|
|
},
|
|
): RoutingDecision {
|
|
if (!options?.learnedModel) {
|
|
return decidePreGenerationRouting(input);
|
|
}
|
|
|
|
const staticDecision = decidePreGenerationRouting(input);
|
|
if (staticDecision.kind === "abstain" || staticDecision.kind === "template-only") {
|
|
return staticDecision;
|
|
}
|
|
|
|
return decideLearnedRouting(input, options.learnedModel);
|
|
}
|
|
|
|
export function routerDecisionKind(decision: RoutingDecision): RoutingDecisionKind {
|
|
return decisionKindFromDecision(decision);
|
|
}
|