import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types'; import type { GarnetClient } from '@common/garnet'; import type { NatsClient } from '@common/nats'; import type { EscalationHistoryService } from '../../memory/escalation-history.js'; import type { MemoryService } from '../../memory/memory-types.js'; import { buildLongTermSummary, shouldRefreshLongTermSummary, } from '../../memory/long-term-summarizer.js'; import { getContext, getRequestLogger } from '@obs/context'; import type { LlmClient } from '@llm/llm-client'; import type { RoomContextService } from '@room/room-context-service'; import type { Skill } from '@skills/skill'; import type { SkillCapabilityCard } from '@skills/skill-loader'; import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus'; import { anonymizeProcessTrace } from '@eval/anonymize'; import { createOnboardingToolRegistry } from './tools.js'; import type { ToolRegistry } from '@tools/registry'; import type { Tool, ToolContext } from '@tools/tool'; import { getCompletedMessage, getGenericHelpMessage, getNicknameInvalidMessage, getOnboardingTemplate, getPhotoFailedMessage, getProgressMessage, getTimeoutWarningMessage, } from './onboarding-templates.js'; import { getAgentAbstentionClarifyThreshold, getAgentAbstentionEnabled, getAgentAbstentionThreshold, getAgentRouterMode, getAgentRouterWeightsPath, getAgentUserMemoryEnabled, getOnboardingSessionKeyPrefix, } from '../../config.js'; import { OnboardingBlackboardStore } from './state/blackboard.js'; import { UserMemoryStore } from './state/user-memory.js'; import { deriveState } from './state/derive.js'; import { OnboardingIntentClassifier } from './state/classifier.js'; import type { ClassifiedIntent, DerivedState, Intent, } from './state/onboarding-state.js'; import { OnboardingOrchestrator } from './runtime/orchestrator.js'; import { decideEarlyAbstention, type AbstentionDecision, } from './runtime/abstention.js'; import { decideHybridRouting, decidePreGenerationRouting, loadLearnedRouterModel, } from './runtime/router.js'; import { getActiveProcessTrace, ProcessTraceCollector, runWithProcessTrace, } from './runtime/trace.js'; import { validateOnboardingResponse } from './runtime/validator.js'; import { getIntentFallbackText } from './runtime/fallback-text-resolver.js'; import { buildLlmMessages } from './runtime/message-builder.js'; import { generateLlmText, type LlmTextResult, } from './runtime/text-generator.js'; import { selectToolsForDerived } from './runtime/tool-selector.js'; import { rememberTurn } from './runtime/state-updater.js'; import { normalizeOnboardingSession } from './session.js'; import type { Blackboard, OnboardingSession, StoredOnboardingSession, } from './types.js'; const CAPABILITY_CARD: SkillCapabilityCard = { canHandle: [ 'new-member', 'nickname-invalid', 'photo-failed', 'progress-update', 'timeout-warning', 'completed', 'onboarding-help', ], cannotHandle: [ 'moderation-report', 'sanctions', 'appeal', 'policy-enforcement', ], domain: 'onboarding', handoffTargets: ['moderation', 'supervisor'], outputContract: { format: 'text', maxChars: '500', requiredFields: 'text', }, riskTier: 'medium', }; export class OnboardingSkill implements Skill { readonly name = 'onboarding'; readonly priority = 70; private readonly classifier: OnboardingIntentClassifier; private readonly blackboardStore: OnboardingBlackboardStore; private readonly orchestrator: OnboardingOrchestrator; private readonly shadowEvalPublisher: ShadowEvalPublisher; private readonly toolRegistry: ToolRegistry; private readonly tools: readonly Tool[]; private readonly userMemoryStore: UserMemoryStore; constructor( private readonly garnetClient: GarnetClient, private readonly llmClient: LlmClient, private readonly roomContextService?: RoomContextService, shadowEvalPublisher?: ShadowEvalPublisher, private readonly natsClient?: Pick, private readonly memoryService?: MemoryService, private readonly escalationHistoryService?: EscalationHistoryService, ) { this.classifier = new OnboardingIntentClassifier(llmClient); this.blackboardStore = new OnboardingBlackboardStore( garnetClient, memoryService?.session, ); this.userMemoryStore = new UserMemoryStore( garnetClient, memoryService?.longTerm, ); this.orchestrator = new OnboardingOrchestrator(llmClient); this.shadowEvalPublisher = shadowEvalPublisher ?? new ShadowEvalBus(garnetClient); this.toolRegistry = createOnboardingToolRegistry(); this.tools = this.toolRegistry.list(); } canHandle(request: AgentRequest): boolean { return request.skill === 'onboarding'; } capabilityCard(): SkillCapabilityCard { return CAPABILITY_CARD; } async handle( request: AgentRequest, conversationHistory: ConversationMessage[], ): Promise { if (getActiveProcessTrace()) { return this.handleWithTrace(request, conversationHistory); } const collector = new ProcessTraceCollector(); return runWithProcessTrace(collector, () => this.handleWithTrace(request, conversationHistory), ); } private async handleWithTrace( request: AgentRequest, conversationHistory: ConversationMessage[], ): Promise { const logger = getRequestLogger(); const trace = getActiveProcessTrace(); const session = await this.loadSession(request.chatroomId, request.userId); if (request.trigger === 'user-message' && !request.userMessage?.trim()) { return this.makeEmptyResponse(request.requestId); } const classifyStep = trace?.startStep('classify'); const classified = await this.classifier.classify(request.userMessage, request.trigger); let classifyFinished = false; const finishClassify = (abstention?: AbstentionDecision): void => { if (classifyFinished) { return; } classifyFinished = true; classifyStep?.finish({ ...(abstention ? { abstention } : {}), confidence: classified.confidence, intent: classified.intent.kind, source: classified.source, }); }; const deriveStep = trace?.startStep('derive'); const derived = deriveState(session, request.trigger, classified.intent); const escalationHistory = await this.loadEscalationHistory(request.userId); const routerMode = getAgentRouterMode(); if (routerMode === 'static' || routerMode === 'learned-v1') { const routerInput = { classified, derived, intentRiskTier: this.capabilityCard().riskTier, recentLatencyP95Ms: this.getRecentLatencyP95Ms(request), request, session, userEscalationCount: escalationHistory?.totalEscalations ?? 0, }; derived.routingDecision = routerMode === 'learned-v1' ? decideHybridRouting(routerInput, { learnedModel: loadLearnedRouterModel(getAgentRouterWeightsPath()), }) : decidePreGenerationRouting(routerInput); } deriveStep?.finish({ phase: derived.phase.kind, responseMode: derived.responseMode, routingDecision: derived.routingDecision, shouldRespond: derived.shouldRespond, tier: derived.tier, }); if (getAgentAbstentionEnabled()) { const decision = await this.decideEarlyAbstention(request, classified, derived); derived.abstentionScore = decision.score; finishClassify(decision); if (decision.action.kind === 'no-response') { const response = this.makeEmptyResponse(request.requestId); trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse('[NO_RESPONSE]', derived, { allowShortResponse: true, }); this.shadowEvalPublisher.publish({ classified, createdAt: new Date().toISOString(), derived, latencyMs: this.getCurrentLatencyMs(), process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, skill: this.name, }); return response; } if (decision.action.kind === 'clarify') { const response = this.makeTemplateResponse( request.requestId, getOnboardingTemplate(decision.action.templateKey), ); trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse(response.text, derived, { allowShortResponse: true, }); await this.rememberTurn(request, derived).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard'); }); await this.observeUserMemory(request, derived, response).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory'); }); this.shadowEvalPublisher.publish({ classified, createdAt: new Date().toISOString(), derived, latencyMs: this.getCurrentLatencyMs(), process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, skill: this.name, }); return response; } } else { finishClassify(); } if (!derived.shouldRespond) { const response = this.makeEmptyResponse(request.requestId); trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse('[NO_RESPONSE]', derived, { allowShortResponse: true, }); return response; } let response: AgentResponse; if (derived.responseMode === 'template') { response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived); } else { const llmResult = await this.handleLlm(request, session, conversationHistory, derived); if (llmResult.status === 'success') { response = this.makeLlmResponse(request.requestId, llmResult.text); } else if (llmResult.status === 'no_response') { response = this.makeEmptyResponse(request.requestId); } else { response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived); } } trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, { allowShortResponse: derived.allowShortResponse, }); if (derived.intent.kind === 'discord-verify-question') { logger.debug( { responseSource: response.source, responseText: response.text, }, '[OnboardingSkill] Final discord verification response', ); } if (response.text.trim()) { await this.rememberTurn(request, derived).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard'); }); await this.observeUserMemory(request, derived, response).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory'); }); this.shadowEvalPublisher.publish({ classified, createdAt: new Date().toISOString(), derived, latencyMs: this.getCurrentLatencyMs(), process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, skill: this.name, }); } return response; } private async loadSession(chatroomId: string, userId: string): Promise { const prefix = getOnboardingSessionKeyPrefix(); const key = `${prefix}:onboarding:${chatroomId}:${userId}`; const rawSession = await this.garnetClient.getJson(key); return normalizeOnboardingSession(rawSession); } private async loadEscalationHistory(userId: string) { if (!this.escalationHistoryService) { return null; } try { return await this.escalationHistoryService.load(userId); } catch (error) { getRequestLogger().warn( { err: error }, '[OnboardingSkill] Failed to load escalation history', ); return null; } } private getRecentLatencyP95Ms(request: AgentRequest): number | undefined { const raw = request.context?.['recentLatencyP95Ms']; return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined; } private async decideEarlyAbstention( request: AgentRequest, classified: ClassifiedIntent, derived: DerivedState, ): Promise { const logger = getRequestLogger(); let blackboard: Blackboard | null = null; try { blackboard = await this.blackboardStore.load(request.chatroomId, request.userId); } catch (error) { logger.warn({ err: error }, '[OnboardingSkill] Failed to load blackboard for abstention'); } return decideEarlyAbstention({ blackboard, classified, derived, request, thresholds: { abstain: getAgentAbstentionThreshold(), clarify: getAgentAbstentionClarifyThreshold(), }, }); } private async handleLlm( request: AgentRequest, session: OnboardingSession | null, conversationHistory: ConversationMessage[], derived: DerivedState, ): Promise { if (!this.llmClient.hasEndpoints()) { return { status: 'failed' }; } const messages = await buildLlmMessages( { blackboardStore: this.blackboardStore, garnetClient: this.garnetClient, memoryService: this.memoryService, roomContextService: this.roomContextService, userMemoryStore: this.userMemoryStore, }, { conversationHistory, derived, request, session, }, ); return generateLlmText(this.orchestrator, messages, derived, { allowShortResponse: derived.allowShortResponse, maxTokens: derived.suggestedMaxTokens, request, retryOnLowQuality: request.trigger === 'user-message' && !derived.allowShortResponse && derived.responseMode !== 'llm-with-tools', toolContext: this.buildToolContext(request, session), tools: selectToolsForDerived(derived, this.tools, this.toolRegistry), }); } private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise { return rememberTurn(this.blackboardStore, request, derived); } private async observeUserMemory( request: AgentRequest, derived: DerivedState, response: AgentResponse, ): Promise { if (!getAgentUserMemoryEnabled()) { return; } const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, { flaggedIssues: derived.phase.kind === 'nickname-rejected' ? derived.phase.issues.map((issue) => issue.message) : [], }); if (!shouldRefreshLongTermSummary(profile)) { return; } void this.refreshLongTermSummary(request, profile); } private async refreshLongTermSummary( request: AgentRequest, profile: Awaited>, ): Promise { try { const conversationSummary = (await this.memoryService?.session.getSummary( request.chatroomId, request.userId, )) ?? null; const longTermSummary = await buildLongTermSummary( this.llmClient, profile, conversationSummary, ); await this.userMemoryStore.updateProfile(request.userId, { longTermSummary, }); } catch (error) { getRequestLogger().warn( { err: error }, '[OnboardingSkill] Failed to refresh long-term user summary', ); } } private makeTemplateFallbackResponse( request: AgentRequest, session: OnboardingSession | null, intent: Intent, derived: DerivedState, ): AgentResponse { switch (request.trigger) { case 'new-member': return this.makeEmptyResponse(request.requestId); case 'nickname-invalid': return this.makeTemplateResponse( request.requestId, getNicknameInvalidMessage(request.nickname, session), ); case 'photo-failed': return this.makeTemplateResponse( request.requestId, getPhotoFailedMessage(request.nickname), ); case 'progress-update': return this.makeTemplateResponse( request.requestId, session ? getProgressMessage(request.nickname, session) : getGenericHelpMessage(request.nickname, session), ); case 'timeout-warning': return this.makeTemplateResponse( request.requestId, getTimeoutWarningMessage(request.nickname), ); case 'completed': return this.makeTemplateResponse(request.requestId, getCompletedMessage(request.nickname)); default: return this.makeTemplateResponse( request.requestId, getIntentFallbackText(request.nickname, session, intent, derived), ); } } private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext { return { chatroomId: request.chatroomId, natsClient: this.natsClient, userId: request.userId, session, garnetClient: this.garnetClient, memoryService: this.memoryService, roomContextService: this.roomContextService, toolRegistry: this.toolRegistry, }; } private makeTemplateResponse(requestId: string, text: string): AgentResponse { return { requestId, text, confidence: 0.8, source: 'template', }; } private makeLlmResponse(requestId: string, text: string): AgentResponse { return { requestId, text, confidence: 0.9, source: 'llm', }; } private makeEmptyResponse(requestId: string): AgentResponse { return { requestId, text: '', confidence: 0, source: 'template', }; } private getCurrentLatencyMs(): number | undefined { const startedAt = getContext()?.startedAt; const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN; if (!Number.isFinite(startedTimestamp)) { return undefined; } return Math.max(0, Date.now() - startedTimestamp); } }