Add skill governance and runtime routing

This commit is contained in:
dahyeon committed 2026-05-14 14:28:59 -07:00
1 parent 6f09408423
commit 41dcfcdef7
71 files changed
+7057 -102

No files matched your search

+243 -24
View File
@@ -9,9 +9,15 @@ import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { meetsTrustTier } from '../../eval/governance/trust.js';
import { createBuiltInToolRegistry } from '@tools/builtins/index';
import type { ToolRegistry } from '@tools/registry';
import type { Tool, ToolContext } from '@tools/tool';
import { PromptBuilder } from './prompts/builder.js';
import { createToolSearchTool } from '@tools/tool-search';
import {
loadPromptVariantManifest,
PromptBuilder,
} from './prompts/builder.js';
import {
getAcknowledgementMessage,
getAgeRequirementMessage,
@@ -23,21 +29,45 @@ import {
getHeartLocationMessage,
getNicknameInvalidMessage,
getNicknameFormatMessage,
getOnboardingTemplate,
getPhotoFailedMessage,
getProgressMessage,
getRemainingStatusMessage,
getTimeoutWarningMessage,
} from './onboarding-templates.js';
import {
getAgentAbstentionClarifyThreshold,
getAgentAbstentionEnabled,
getAgentAbstentionThreshold,
getAgentRouterMode,
getAgentRouterWeightsPath,
getAgentTrustMinTier,
getAgentToolSearchEnabled,
getAgentToolSearchThreshold,
getAgentUserMemoryEnabled,
getOnboardingSessionKeyPrefix,
getLlmTemperature,
getPromptVariantId,
} from '../../config.js';
import { OnboardingBlackboardStore } from './state/blackboard.js';
import { UserMemoryStore } from './state/user-memory.js';
import { deriveState } from './state/derive.js';
import { OnboardingIntentClassifier } from './state/classifier.js';
import type { DerivedState, Intent } from './state/onboarding-state.js';
import type {
ClassifiedIntent,
DerivedState,
Intent,
} from './state/onboarding-state.js';
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
import {
decideEarlyAbstention,
type AbstentionDecision,
} from './runtime/abstention.js';
import {
decideHybridRouting,
decidePreGenerationRouting,
loadLearnedRouterModel,
} from './runtime/router.js';
import {
getActiveProcessTrace,
ProcessTraceCollector,
@@ -45,7 +75,12 @@ import {
} from './runtime/trace.js';
import { validateOnboardingResponse } from './runtime/validator.js';
import { normalizeOnboardingSession } from './session.js';
import type { OnboardingSession, PromptContext, StoredOnboardingSession } from './types.js';
import type {
Blackboard,
OnboardingSession,
PromptContext,
StoredOnboardingSession,
} from './types.js';
type LlmTextResult =
| { status: 'success'; text: string }
@@ -58,7 +93,9 @@ export class OnboardingSkill implements Skill {
private readonly blackboardStore: OnboardingBlackboardStore;
private readonly orchestrator: OnboardingOrchestrator;
private readonly shadowEvalPublisher: ShadowEvalPublisher;
private readonly toolRegistry: ToolRegistry;
private readonly tools: readonly Tool<unknown, unknown>[];
private readonly userMemoryStore: UserMemoryStore;
constructor(
private readonly garnetClient: GarnetClient,
@@ -69,9 +106,11 @@ export class OnboardingSkill implements Skill {
) {
this.classifier = new OnboardingIntentClassifier(llmClient);
this.blackboardStore = new OnboardingBlackboardStore(garnetClient);
this.userMemoryStore = new UserMemoryStore(garnetClient);
this.orchestrator = new OnboardingOrchestrator(llmClient);
this.shadowEvalPublisher = shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
this.tools = createBuiltInToolRegistry().list();
this.toolRegistry = createBuiltInToolRegistry();
this.tools = this.toolRegistry.list();
}
canHandle(request: AgentRequest): boolean {
@@ -106,21 +145,109 @@ export class OnboardingSkill implements Skill {
const classifyStep = trace?.startStep('classify');
const classified = await this.classifier.classify(request.userMessage, request.trigger);
classifyStep?.finish({
confidence: classified.confidence,
intent: classified.intent.kind,
source: classified.source,
});
let classifyFinished = false;
const finishClassify = (abstention?: AbstentionDecision): void => {
if (classifyFinished) {
return;
}
classifyFinished = true;
classifyStep?.finish({
...(abstention ? { abstention } : {}),
confidence: classified.confidence,
intent: classified.intent.kind,
source: classified.source,
});
};
const deriveStep = trace?.startStep('derive');
const derived = deriveState(session, request.trigger, classified.intent);
const routerMode = getAgentRouterMode();
if (routerMode === 'static' || routerMode === 'learned-v1') {
const routerInput = {
classified,
derived,
request,
session,
};
derived.routingDecision =
routerMode === 'learned-v1'
? decideHybridRouting(routerInput, {
learnedModel: loadLearnedRouterModel(getAgentRouterWeightsPath()),
})
: decidePreGenerationRouting(routerInput);
}
deriveStep?.finish({
phase: derived.phase.kind,
responseMode: derived.responseMode,
routingDecision: derived.routingDecision,
shouldRespond: derived.shouldRespond,
tier: derived.tier,
});
if (getAgentAbstentionEnabled()) {
const decision = await this.decideEarlyAbstention(request, classified, derived);
derived.abstentionScore = decision.score;
finishClassify(decision);
if (decision.action.kind === 'no-response') {
const response = this.makeEmptyResponse(request.requestId);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse('[NO_RESPONSE]', derived, {
allowShortResponse: true,
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
});
return response;
}
if (decision.action.kind === 'clarify') {
const response = this.makeTemplateResponse(
request.requestId,
getOnboardingTemplate(decision.action.templateKey),
);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse(response.text, derived, {
allowShortResponse: true,
});
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
});
await this.observeUserMemory(request, derived, response).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
});
return response;
}
} else {
finishClassify();
}
if (!derived.shouldRespond) {
const response = this.makeEmptyResponse(request.requestId);
trace?.startStep('respond').finish({
@@ -173,6 +300,9 @@ export class OnboardingSkill implements Skill {
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
});
await this.observeUserMemory(request, derived, response).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
});
this.shadowEvalPublisher.publish({
classified,
@@ -195,6 +325,32 @@ export class OnboardingSkill implements Skill {
return normalizeOnboardingSession(rawSession);
}
private async decideEarlyAbstention(
request: AgentRequest,
classified: ClassifiedIntent,
derived: DerivedState,
): Promise<AbstentionDecision> {
const logger = getRequestLogger();
let blackboard: Blackboard | null = null;
try {
blackboard = await this.blackboardStore.load(request.chatroomId, request.userId);
} catch (error) {
logger.warn({ err: error }, '[OnboardingSkill] Failed to load blackboard for abstention');
}
return decideEarlyAbstention({
blackboard,
classified,
derived,
request,
thresholds: {
abstain: getAgentAbstentionThreshold(),
clarify: getAgentAbstentionClarifyThreshold(),
},
});
}
private async handleLlm(
request: AgentRequest,
session: OnboardingSession | null,
@@ -206,6 +362,7 @@ export class OnboardingSkill implements Skill {
return this.generateLlmText(messages, derived, {
allowShortResponse: derived.allowShortResponse,
maxTokens: derived.suggestedMaxTokens,
request,
retryOnLowQuality:
request.trigger === 'user-message' &&
!derived.allowShortResponse &&
@@ -229,10 +386,17 @@ export class OnboardingSkill implements Skill {
return undefined;
})
: Promise.resolve(undefined);
const [blackboard, summary, roomContext] = await Promise.all([
const userProfilePromise = getAgentUserMemoryEnabled()
? this.userMemoryStore.load(request.userId).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory');
return null;
})
: Promise.resolve(null);
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
this.blackboardStore.load(request.chatroomId, request.userId),
this.loadConversationSummary(request.chatroomId, request.userId),
roomContextPromise,
userProfilePromise,
]);
const context: PromptContext = {
@@ -241,11 +405,21 @@ export class OnboardingSkill implements Skill {
roomContext,
summary,
userMessage: this.buildPromptUserMessage(request, derived),
userProfile,
};
const builder = PromptBuilder.forDerived(derived, session, context);
const promptVariantId = getPromptVariantId();
return promptVariantId ? builder.applyVariant(promptVariantId).build() : builder.build();
if (!promptVariantId) {
return builder.build();
}
const manifest = loadPromptVariantManifest(promptVariantId);
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
return builder.build();
}
return builder.applyVariant(manifest).build();
}
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
@@ -283,6 +457,10 @@ export class OnboardingSkill implements Skill {
const userMessage = request.userMessage?.trim();
await this.blackboardStore.update(request.chatroomId, request.userId, (current) => {
const sameIntentStreak =
current.lastIntent === derived.intent.kind
? (current.sameIntentStreak ?? 1) + 1
: 1;
const openQuestions =
userMessage &&
(derived.intent.kind === 'help-how' ||
@@ -311,6 +489,7 @@ export class OnboardingSkill implements Skill {
lastIntentAt: now,
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
recentNicknameAttempts,
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
toneMemory: {
...current.toneMemory,
greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt,
@@ -320,6 +499,23 @@ export class OnboardingSkill implements Skill {
});
}
private async observeUserMemory(
request: AgentRequest,
derived: DerivedState,
response: AgentResponse,
): Promise<void> {
if (!getAgentUserMemoryEnabled()) {
return;
}
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: [],
});
}
private makeTemplateFallbackResponse(
request: AgentRequest,
session: OnboardingSession | null,
@@ -427,6 +623,7 @@ export class OnboardingSkill implements Skill {
options?: {
allowShortResponse?: boolean;
maxTokens?: number;
request: AgentRequest;
retryOnLowQuality?: boolean;
temperature?: number;
toolContext?: ToolContext | undefined;
@@ -482,6 +679,7 @@ export class OnboardingSkill implements Skill {
const toolContext = options?.toolContext ?? {};
const llmText = await this.orchestrator.run(derived, messages, {
maxTokens: requestOptions.maxTokens,
request: options?.request,
temperature: requestOptions.temperature,
toolContext,
tools: options?.tools,
@@ -519,34 +717,55 @@ export class OnboardingSkill implements Skill {
session,
garnetClient: this.garnetClient,
roomContextService: this.roomContextService,
toolRegistry: this.toolRegistry,
};
}
private selectToolsForDerived(derived: DerivedState): readonly Tool<unknown, unknown>[] {
const maybeUseToolSearch = (
tools: readonly Tool<unknown, unknown>[],
): readonly Tool<unknown, unknown>[] => {
if (
getAgentToolSearchEnabled() &&
tools.length >= getAgentToolSearchThreshold()
) {
return [createToolSearchTool(this.toolRegistry)];
}
return tools;
};
if (derived.intent.kind === 'discord-verify-question') {
return this.tools.filter((tool) => tool.name === 'verify-discord-nickname');
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'verify-discord-nickname'),
);
}
if (derived.phase.kind === 'nickname-rejected') {
return this.tools.filter(
(tool) =>
tool.name === 'validate-nickname' ||
tool.name === 'lookup-community-rule' ||
tool.name === 'check-onboarding-progress',
return maybeUseToolSearch(
this.tools.filter(
(tool) =>
tool.name === 'validate-nickname' ||
tool.name === 'lookup-community-rule' ||
tool.name === 'check-onboarding-progress',
),
);
}
if (derived.intent.kind === 'status-question') {
return this.tools.filter((tool) => tool.name === 'check-onboarding-progress');
}
if (derived.intent.kind === 'discord-question') {
return this.tools.filter(
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'check-onboarding-progress'),
);
}
return this.tools;
if (derived.intent.kind === 'discord-question') {
return maybeUseToolSearch(
this.tools.filter(
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
),
);
}
return maybeUseToolSearch(this.tools);
}
private makeTemplateResponse(requestId: string, text: string): AgentResponse {