Add skill governance and runtime routing
This commit is contained in:
1 parent
6f09408423
commit
41dcfcdef7
71 files changed
+7057
-102
No files matched your search
+243
-24
@@ -9,9 +9,15 @@ import type { RoomContextService } from '@room/room-context-service';
|
||||
import type { Skill } from '@skills/skill';
|
||||
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
||||
import { anonymizeProcessTrace } from '@eval/anonymize';
|
||||
import { meetsTrustTier } from '../../eval/governance/trust.js';
|
||||
import { createBuiltInToolRegistry } from '@tools/builtins/index';
|
||||
import type { ToolRegistry } from '@tools/registry';
|
||||
import type { Tool, ToolContext } from '@tools/tool';
|
||||
import { PromptBuilder } from './prompts/builder.js';
|
||||
import { createToolSearchTool } from '@tools/tool-search';
|
||||
import {
|
||||
loadPromptVariantManifest,
|
||||
PromptBuilder,
|
||||
} from './prompts/builder.js';
|
||||
import {
|
||||
getAcknowledgementMessage,
|
||||
getAgeRequirementMessage,
|
||||
@@ -23,21 +29,45 @@ import {
|
||||
getHeartLocationMessage,
|
||||
getNicknameInvalidMessage,
|
||||
getNicknameFormatMessage,
|
||||
getOnboardingTemplate,
|
||||
getPhotoFailedMessage,
|
||||
getProgressMessage,
|
||||
getRemainingStatusMessage,
|
||||
getTimeoutWarningMessage,
|
||||
} from './onboarding-templates.js';
|
||||
import {
|
||||
getAgentAbstentionClarifyThreshold,
|
||||
getAgentAbstentionEnabled,
|
||||
getAgentAbstentionThreshold,
|
||||
getAgentRouterMode,
|
||||
getAgentRouterWeightsPath,
|
||||
getAgentTrustMinTier,
|
||||
getAgentToolSearchEnabled,
|
||||
getAgentToolSearchThreshold,
|
||||
getAgentUserMemoryEnabled,
|
||||
getOnboardingSessionKeyPrefix,
|
||||
getLlmTemperature,
|
||||
getPromptVariantId,
|
||||
} from '../../config.js';
|
||||
import { OnboardingBlackboardStore } from './state/blackboard.js';
|
||||
import { UserMemoryStore } from './state/user-memory.js';
|
||||
import { deriveState } from './state/derive.js';
|
||||
import { OnboardingIntentClassifier } from './state/classifier.js';
|
||||
import type { DerivedState, Intent } from './state/onboarding-state.js';
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
Intent,
|
||||
} from './state/onboarding-state.js';
|
||||
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
|
||||
import {
|
||||
decideEarlyAbstention,
|
||||
type AbstentionDecision,
|
||||
} from './runtime/abstention.js';
|
||||
import {
|
||||
decideHybridRouting,
|
||||
decidePreGenerationRouting,
|
||||
loadLearnedRouterModel,
|
||||
} from './runtime/router.js';
|
||||
import {
|
||||
getActiveProcessTrace,
|
||||
ProcessTraceCollector,
|
||||
@@ -45,7 +75,12 @@ import {
|
||||
} from './runtime/trace.js';
|
||||
import { validateOnboardingResponse } from './runtime/validator.js';
|
||||
import { normalizeOnboardingSession } from './session.js';
|
||||
import type { OnboardingSession, PromptContext, StoredOnboardingSession } from './types.js';
|
||||
import type {
|
||||
Blackboard,
|
||||
OnboardingSession,
|
||||
PromptContext,
|
||||
StoredOnboardingSession,
|
||||
} from './types.js';
|
||||
|
||||
type LlmTextResult =
|
||||
| { status: 'success'; text: string }
|
||||
@@ -58,7 +93,9 @@ export class OnboardingSkill implements Skill {
|
||||
private readonly blackboardStore: OnboardingBlackboardStore;
|
||||
private readonly orchestrator: OnboardingOrchestrator;
|
||||
private readonly shadowEvalPublisher: ShadowEvalPublisher;
|
||||
private readonly toolRegistry: ToolRegistry;
|
||||
private readonly tools: readonly Tool<unknown, unknown>[];
|
||||
private readonly userMemoryStore: UserMemoryStore;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
@@ -69,9 +106,11 @@ export class OnboardingSkill implements Skill {
|
||||
) {
|
||||
this.classifier = new OnboardingIntentClassifier(llmClient);
|
||||
this.blackboardStore = new OnboardingBlackboardStore(garnetClient);
|
||||
this.userMemoryStore = new UserMemoryStore(garnetClient);
|
||||
this.orchestrator = new OnboardingOrchestrator(llmClient);
|
||||
this.shadowEvalPublisher = shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
|
||||
this.tools = createBuiltInToolRegistry().list();
|
||||
this.toolRegistry = createBuiltInToolRegistry();
|
||||
this.tools = this.toolRegistry.list();
|
||||
}
|
||||
|
||||
canHandle(request: AgentRequest): boolean {
|
||||
@@ -106,21 +145,109 @@ export class OnboardingSkill implements Skill {
|
||||
|
||||
const classifyStep = trace?.startStep('classify');
|
||||
const classified = await this.classifier.classify(request.userMessage, request.trigger);
|
||||
classifyStep?.finish({
|
||||
confidence: classified.confidence,
|
||||
intent: classified.intent.kind,
|
||||
source: classified.source,
|
||||
});
|
||||
let classifyFinished = false;
|
||||
const finishClassify = (abstention?: AbstentionDecision): void => {
|
||||
if (classifyFinished) {
|
||||
return;
|
||||
}
|
||||
classifyFinished = true;
|
||||
classifyStep?.finish({
|
||||
...(abstention ? { abstention } : {}),
|
||||
confidence: classified.confidence,
|
||||
intent: classified.intent.kind,
|
||||
source: classified.source,
|
||||
});
|
||||
};
|
||||
|
||||
const deriveStep = trace?.startStep('derive');
|
||||
const derived = deriveState(session, request.trigger, classified.intent);
|
||||
const routerMode = getAgentRouterMode();
|
||||
if (routerMode === 'static' || routerMode === 'learned-v1') {
|
||||
const routerInput = {
|
||||
classified,
|
||||
derived,
|
||||
request,
|
||||
session,
|
||||
};
|
||||
derived.routingDecision =
|
||||
routerMode === 'learned-v1'
|
||||
? decideHybridRouting(routerInput, {
|
||||
learnedModel: loadLearnedRouterModel(getAgentRouterWeightsPath()),
|
||||
})
|
||||
: decidePreGenerationRouting(routerInput);
|
||||
}
|
||||
deriveStep?.finish({
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
routingDecision: derived.routingDecision,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
if (getAgentAbstentionEnabled()) {
|
||||
const decision = await this.decideEarlyAbstention(request, classified, derived);
|
||||
derived.abstentionScore = decision.score;
|
||||
finishClassify(decision);
|
||||
|
||||
if (decision.action.kind === 'no-response') {
|
||||
const response = this.makeEmptyResponse(request.requestId);
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse('[NO_RESPONSE]', derived, {
|
||||
allowShortResponse: true,
|
||||
});
|
||||
this.shadowEvalPublisher.publish({
|
||||
classified,
|
||||
createdAt: new Date().toISOString(),
|
||||
derived,
|
||||
latencyMs: this.getCurrentLatencyMs(),
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
if (decision.action.kind === 'clarify') {
|
||||
const response = this.makeTemplateResponse(
|
||||
request.requestId,
|
||||
getOnboardingTemplate(decision.action.templateKey),
|
||||
);
|
||||
trace?.startStep('respond').finish({
|
||||
draftText: response.text,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
});
|
||||
validateOnboardingResponse(response.text, derived, {
|
||||
allowShortResponse: true,
|
||||
});
|
||||
|
||||
await this.rememberTurn(request, derived).catch((error) => {
|
||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
|
||||
});
|
||||
await this.observeUserMemory(request, derived, response).catch((error) => {
|
||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
|
||||
});
|
||||
|
||||
this.shadowEvalPublisher.publish({
|
||||
classified,
|
||||
createdAt: new Date().toISOString(),
|
||||
derived,
|
||||
latencyMs: this.getCurrentLatencyMs(),
|
||||
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
|
||||
return response;
|
||||
}
|
||||
} else {
|
||||
finishClassify();
|
||||
}
|
||||
|
||||
if (!derived.shouldRespond) {
|
||||
const response = this.makeEmptyResponse(request.requestId);
|
||||
trace?.startStep('respond').finish({
|
||||
@@ -173,6 +300,9 @@ export class OnboardingSkill implements Skill {
|
||||
await this.rememberTurn(request, derived).catch((error) => {
|
||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
|
||||
});
|
||||
await this.observeUserMemory(request, derived, response).catch((error) => {
|
||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
|
||||
});
|
||||
|
||||
this.shadowEvalPublisher.publish({
|
||||
classified,
|
||||
@@ -195,6 +325,32 @@ export class OnboardingSkill implements Skill {
|
||||
return normalizeOnboardingSession(rawSession);
|
||||
}
|
||||
|
||||
private async decideEarlyAbstention(
|
||||
request: AgentRequest,
|
||||
classified: ClassifiedIntent,
|
||||
derived: DerivedState,
|
||||
): Promise<AbstentionDecision> {
|
||||
const logger = getRequestLogger();
|
||||
let blackboard: Blackboard | null = null;
|
||||
|
||||
try {
|
||||
blackboard = await this.blackboardStore.load(request.chatroomId, request.userId);
|
||||
} catch (error) {
|
||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to load blackboard for abstention');
|
||||
}
|
||||
|
||||
return decideEarlyAbstention({
|
||||
blackboard,
|
||||
classified,
|
||||
derived,
|
||||
request,
|
||||
thresholds: {
|
||||
abstain: getAgentAbstentionThreshold(),
|
||||
clarify: getAgentAbstentionClarifyThreshold(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async handleLlm(
|
||||
request: AgentRequest,
|
||||
session: OnboardingSession | null,
|
||||
@@ -206,6 +362,7 @@ export class OnboardingSkill implements Skill {
|
||||
return this.generateLlmText(messages, derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
maxTokens: derived.suggestedMaxTokens,
|
||||
request,
|
||||
retryOnLowQuality:
|
||||
request.trigger === 'user-message' &&
|
||||
!derived.allowShortResponse &&
|
||||
@@ -229,10 +386,17 @@ export class OnboardingSkill implements Skill {
|
||||
return undefined;
|
||||
})
|
||||
: Promise.resolve(undefined);
|
||||
const [blackboard, summary, roomContext] = await Promise.all([
|
||||
const userProfilePromise = getAgentUserMemoryEnabled()
|
||||
? this.userMemoryStore.load(request.userId).catch((error) => {
|
||||
logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory');
|
||||
return null;
|
||||
})
|
||||
: Promise.resolve(null);
|
||||
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
|
||||
this.blackboardStore.load(request.chatroomId, request.userId),
|
||||
this.loadConversationSummary(request.chatroomId, request.userId),
|
||||
roomContextPromise,
|
||||
userProfilePromise,
|
||||
]);
|
||||
|
||||
const context: PromptContext = {
|
||||
@@ -241,11 +405,21 @@ export class OnboardingSkill implements Skill {
|
||||
roomContext,
|
||||
summary,
|
||||
userMessage: this.buildPromptUserMessage(request, derived),
|
||||
userProfile,
|
||||
};
|
||||
|
||||
const builder = PromptBuilder.forDerived(derived, session, context);
|
||||
const promptVariantId = getPromptVariantId();
|
||||
return promptVariantId ? builder.applyVariant(promptVariantId).build() : builder.build();
|
||||
if (!promptVariantId) {
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
const manifest = loadPromptVariantManifest(promptVariantId);
|
||||
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
return builder.applyVariant(manifest).build();
|
||||
}
|
||||
|
||||
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
|
||||
@@ -283,6 +457,10 @@ export class OnboardingSkill implements Skill {
|
||||
const userMessage = request.userMessage?.trim();
|
||||
|
||||
await this.blackboardStore.update(request.chatroomId, request.userId, (current) => {
|
||||
const sameIntentStreak =
|
||||
current.lastIntent === derived.intent.kind
|
||||
? (current.sameIntentStreak ?? 1) + 1
|
||||
: 1;
|
||||
const openQuestions =
|
||||
userMessage &&
|
||||
(derived.intent.kind === 'help-how' ||
|
||||
@@ -311,6 +489,7 @@ export class OnboardingSkill implements Skill {
|
||||
lastIntentAt: now,
|
||||
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
|
||||
recentNicknameAttempts,
|
||||
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
|
||||
toneMemory: {
|
||||
...current.toneMemory,
|
||||
greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt,
|
||||
@@ -320,6 +499,23 @@ export class OnboardingSkill implements Skill {
|
||||
});
|
||||
}
|
||||
|
||||
private async observeUserMemory(
|
||||
request: AgentRequest,
|
||||
derived: DerivedState,
|
||||
response: AgentResponse,
|
||||
): Promise<void> {
|
||||
if (!getAgentUserMemoryEnabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
||||
flaggedIssues:
|
||||
derived.phase.kind === 'nickname-rejected'
|
||||
? derived.phase.issues.map((issue) => issue.message)
|
||||
: [],
|
||||
});
|
||||
}
|
||||
|
||||
private makeTemplateFallbackResponse(
|
||||
request: AgentRequest,
|
||||
session: OnboardingSession | null,
|
||||
@@ -427,6 +623,7 @@ export class OnboardingSkill implements Skill {
|
||||
options?: {
|
||||
allowShortResponse?: boolean;
|
||||
maxTokens?: number;
|
||||
request: AgentRequest;
|
||||
retryOnLowQuality?: boolean;
|
||||
temperature?: number;
|
||||
toolContext?: ToolContext | undefined;
|
||||
@@ -482,6 +679,7 @@ export class OnboardingSkill implements Skill {
|
||||
const toolContext = options?.toolContext ?? {};
|
||||
const llmText = await this.orchestrator.run(derived, messages, {
|
||||
maxTokens: requestOptions.maxTokens,
|
||||
request: options?.request,
|
||||
temperature: requestOptions.temperature,
|
||||
toolContext,
|
||||
tools: options?.tools,
|
||||
@@ -519,34 +717,55 @@ export class OnboardingSkill implements Skill {
|
||||
session,
|
||||
garnetClient: this.garnetClient,
|
||||
roomContextService: this.roomContextService,
|
||||
toolRegistry: this.toolRegistry,
|
||||
};
|
||||
}
|
||||
|
||||
private selectToolsForDerived(derived: DerivedState): readonly Tool<unknown, unknown>[] {
|
||||
const maybeUseToolSearch = (
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
): readonly Tool<unknown, unknown>[] => {
|
||||
if (
|
||||
getAgentToolSearchEnabled() &&
|
||||
tools.length >= getAgentToolSearchThreshold()
|
||||
) {
|
||||
return [createToolSearchTool(this.toolRegistry)];
|
||||
}
|
||||
return tools;
|
||||
};
|
||||
|
||||
if (derived.intent.kind === 'discord-verify-question') {
|
||||
return this.tools.filter((tool) => tool.name === 'verify-discord-nickname');
|
||||
return maybeUseToolSearch(
|
||||
this.tools.filter((tool) => tool.name === 'verify-discord-nickname'),
|
||||
);
|
||||
}
|
||||
|
||||
if (derived.phase.kind === 'nickname-rejected') {
|
||||
return this.tools.filter(
|
||||
(tool) =>
|
||||
tool.name === 'validate-nickname' ||
|
||||
tool.name === 'lookup-community-rule' ||
|
||||
tool.name === 'check-onboarding-progress',
|
||||
return maybeUseToolSearch(
|
||||
this.tools.filter(
|
||||
(tool) =>
|
||||
tool.name === 'validate-nickname' ||
|
||||
tool.name === 'lookup-community-rule' ||
|
||||
tool.name === 'check-onboarding-progress',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (derived.intent.kind === 'status-question') {
|
||||
return this.tools.filter((tool) => tool.name === 'check-onboarding-progress');
|
||||
}
|
||||
|
||||
if (derived.intent.kind === 'discord-question') {
|
||||
return this.tools.filter(
|
||||
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
|
||||
return maybeUseToolSearch(
|
||||
this.tools.filter((tool) => tool.name === 'check-onboarding-progress'),
|
||||
);
|
||||
}
|
||||
|
||||
return this.tools;
|
||||
if (derived.intent.kind === 'discord-question') {
|
||||
return maybeUseToolSearch(
|
||||
this.tools.filter(
|
||||
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return maybeUseToolSearch(this.tools);
|
||||
}
|
||||
|
||||
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
|
||||
|
||||
Reference in new issue
Block a user