import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types'; import { getConversationSummaryKey } from '@conv/conversation-store'; import type { GarnetClient } from '@common/garnet'; import type { NatsClient } from '@common/nats'; import { getContext, getRequestLogger } from '@obs/context'; import type { LlmClient } from '@llm/llm-client'; import type { ChatMessage } from '@llm/types'; import type { RoomContextService } from '@room/room-context-service'; import type { Skill } from '@skills/skill'; import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus'; import { anonymizeProcessTrace } from '@eval/anonymize'; import { meetsTrustTier } from '../../eval/governance/trust.js'; import { createBuiltInToolRegistry } from '@tools/builtins/index'; import type { ToolRegistry } from '@tools/registry'; import type { Tool, ToolContext } from '@tools/tool'; import { createToolSearchTool } from '@tools/tool-search'; import { loadPromptVariantManifest, PromptBuilder, } from './prompts/builder.js'; import { getAcknowledgementMessage, getAgeRequirementMessage, getCompletedMessage, getDiscordMessage, getGreetingMessage, getGenericHelpMessage, getHeartGuideMessage, getHeartLocationMessage, getNicknameInvalidMessage, getNicknameFormatMessage, getOnboardingTemplate, getPhotoFailedMessage, getProgressMessage, getRemainingStatusMessage, getTimeoutWarningMessage, } from './onboarding-templates.js'; import { getAgentAbstentionClarifyThreshold, getAgentAbstentionEnabled, getAgentAbstentionThreshold, getAgentRouterMode, getAgentRouterWeightsPath, getAgentTrustMinTier, getAgentToolSearchEnabled, getAgentToolSearchThreshold, getAgentUserMemoryEnabled, getOnboardingSessionKeyPrefix, getLlmTemperature, getPromptVariantId, } from '../../config.js'; import { OnboardingBlackboardStore } from './state/blackboard.js'; import { UserMemoryStore } from './state/user-memory.js'; import { deriveState } from './state/derive.js'; import { OnboardingIntentClassifier } from './state/classifier.js'; import type { ClassifiedIntent, DerivedState, Intent, } from './state/onboarding-state.js'; import { OnboardingOrchestrator } from './runtime/orchestrator.js'; import { decideEarlyAbstention, type AbstentionDecision, } from './runtime/abstention.js'; import { decideHybridRouting, decidePreGenerationRouting, loadLearnedRouterModel, } from './runtime/router.js'; import { getActiveProcessTrace, ProcessTraceCollector, runWithProcessTrace, } from './runtime/trace.js'; import { validateOnboardingResponse } from './runtime/validator.js'; import { normalizeOnboardingSession } from './session.js'; import type { Blackboard, OnboardingSession, PromptContext, StoredOnboardingSession, } from './types.js'; type LlmTextResult = | { status: 'success'; text: string } | { status: 'no_response' } | { status: 'failed' }; export class OnboardingSkill implements Skill { readonly name = 'onboarding'; private readonly classifier: OnboardingIntentClassifier; private readonly blackboardStore: OnboardingBlackboardStore; private readonly orchestrator: OnboardingOrchestrator; private readonly shadowEvalPublisher: ShadowEvalPublisher; private readonly toolRegistry: ToolRegistry; private readonly tools: readonly Tool[]; private readonly userMemoryStore: UserMemoryStore; constructor( private readonly garnetClient: GarnetClient, private readonly llmClient: LlmClient, private readonly roomContextService?: RoomContextService, shadowEvalPublisher?: ShadowEvalPublisher, private readonly natsClient?: Pick, ) { this.classifier = new OnboardingIntentClassifier(llmClient); this.blackboardStore = new OnboardingBlackboardStore(garnetClient); this.userMemoryStore = new UserMemoryStore(garnetClient); this.orchestrator = new OnboardingOrchestrator(llmClient); this.shadowEvalPublisher = shadowEvalPublisher ?? new ShadowEvalBus(garnetClient); this.toolRegistry = createBuiltInToolRegistry(); this.tools = this.toolRegistry.list(); } canHandle(request: AgentRequest): boolean { return request.skill === 'onboarding'; } async handle( request: AgentRequest, conversationHistory: ConversationMessage[], ): Promise { if (getActiveProcessTrace()) { return this.handleWithTrace(request, conversationHistory); } const collector = new ProcessTraceCollector(); return runWithProcessTrace(collector, () => this.handleWithTrace(request, conversationHistory), ); } private async handleWithTrace( request: AgentRequest, conversationHistory: ConversationMessage[], ): Promise { const logger = getRequestLogger(); const trace = getActiveProcessTrace(); const session = await this.loadSession(request.chatroomId, request.userId); if (request.trigger === 'user-message' && !request.userMessage?.trim()) { return this.makeEmptyResponse(request.requestId); } const classifyStep = trace?.startStep('classify'); const classified = await this.classifier.classify(request.userMessage, request.trigger); let classifyFinished = false; const finishClassify = (abstention?: AbstentionDecision): void => { if (classifyFinished) { return; } classifyFinished = true; classifyStep?.finish({ ...(abstention ? { abstention } : {}), confidence: classified.confidence, intent: classified.intent.kind, source: classified.source, }); }; const deriveStep = trace?.startStep('derive'); const derived = deriveState(session, request.trigger, classified.intent); const routerMode = getAgentRouterMode(); if (routerMode === 'static' || routerMode === 'learned-v1') { const routerInput = { classified, derived, request, session, }; derived.routingDecision = routerMode === 'learned-v1' ? decideHybridRouting(routerInput, { learnedModel: loadLearnedRouterModel(getAgentRouterWeightsPath()), }) : decidePreGenerationRouting(routerInput); } deriveStep?.finish({ phase: derived.phase.kind, responseMode: derived.responseMode, routingDecision: derived.routingDecision, shouldRespond: derived.shouldRespond, tier: derived.tier, }); if (getAgentAbstentionEnabled()) { const decision = await this.decideEarlyAbstention(request, classified, derived); derived.abstentionScore = decision.score; finishClassify(decision); if (decision.action.kind === 'no-response') { const response = this.makeEmptyResponse(request.requestId); trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse('[NO_RESPONSE]', derived, { allowShortResponse: true, }); this.shadowEvalPublisher.publish({ classified, createdAt: new Date().toISOString(), derived, latencyMs: this.getCurrentLatencyMs(), process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, }); return response; } if (decision.action.kind === 'clarify') { const response = this.makeTemplateResponse( request.requestId, getOnboardingTemplate(decision.action.templateKey), ); trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse(response.text, derived, { allowShortResponse: true, }); await this.rememberTurn(request, derived).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard'); }); await this.observeUserMemory(request, derived, response).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory'); }); this.shadowEvalPublisher.publish({ classified, createdAt: new Date().toISOString(), derived, latencyMs: this.getCurrentLatencyMs(), process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, }); return response; } } else { finishClassify(); } if (!derived.shouldRespond) { const response = this.makeEmptyResponse(request.requestId); trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse('[NO_RESPONSE]', derived, { allowShortResponse: true, }); return response; } let response: AgentResponse; if (derived.responseMode === 'template') { response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived); } else { const llmResult = await this.handleLlm(request, session, conversationHistory, derived); if (llmResult.status === 'success') { response = this.makeLlmResponse(request.requestId, llmResult.text); } else if (llmResult.status === 'no_response') { response = this.makeEmptyResponse(request.requestId); } else { response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived); } } trace?.startStep('respond').finish({ draftText: response.text, tokensIn: 0, tokensOut: 0, }); validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, { allowShortResponse: derived.allowShortResponse, }); if (derived.intent.kind === 'discord-verify-question') { logger.debug( { responseSource: response.source, responseText: response.text, }, '[OnboardingSkill] Final discord verification response', ); } if (response.text.trim()) { await this.rememberTurn(request, derived).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard'); }); await this.observeUserMemory(request, derived, response).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory'); }); this.shadowEvalPublisher.publish({ classified, createdAt: new Date().toISOString(), derived, latencyMs: this.getCurrentLatencyMs(), process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined, request, response, }); } return response; } private async loadSession(chatroomId: string, userId: string): Promise { const prefix = getOnboardingSessionKeyPrefix(); const key = `${prefix}:onboarding:${chatroomId}:${userId}`; const rawSession = await this.garnetClient.getJson(key); return normalizeOnboardingSession(rawSession); } private async decideEarlyAbstention( request: AgentRequest, classified: ClassifiedIntent, derived: DerivedState, ): Promise { const logger = getRequestLogger(); let blackboard: Blackboard | null = null; try { blackboard = await this.blackboardStore.load(request.chatroomId, request.userId); } catch (error) { logger.warn({ err: error }, '[OnboardingSkill] Failed to load blackboard for abstention'); } return decideEarlyAbstention({ blackboard, classified, derived, request, thresholds: { abstain: getAgentAbstentionThreshold(), clarify: getAgentAbstentionClarifyThreshold(), }, }); } private async handleLlm( request: AgentRequest, session: OnboardingSession | null, conversationHistory: ConversationMessage[], derived: DerivedState, ): Promise { const messages = await this.buildLlmMessages(request, session, conversationHistory, derived); return this.generateLlmText(messages, derived, { allowShortResponse: derived.allowShortResponse, maxTokens: derived.suggestedMaxTokens, request, retryOnLowQuality: request.trigger === 'user-message' && !derived.allowShortResponse && derived.responseMode !== 'llm-with-tools', toolContext: this.buildToolContext(request, session), tools: this.selectToolsForDerived(derived), }); } private async buildLlmMessages( request: AgentRequest, session: OnboardingSession | null, conversationHistory: ConversationMessage[], derived: DerivedState, ): Promise { const logger = getRequestLogger(); const roomContextPromise = request.trigger === 'user-message' && this.roomContextService ? this.roomContextService.getRecentMessages(request.chatroomId).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to fetch room context'); return undefined; }) : Promise.resolve(undefined); const userProfilePromise = getAgentUserMemoryEnabled() ? this.userMemoryStore.load(request.userId).catch((error) => { logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory'); return null; }) : Promise.resolve(null); const [blackboard, summary, roomContext, userProfile] = await Promise.all([ this.blackboardStore.load(request.chatroomId, request.userId), this.loadConversationSummary(request.chatroomId, request.userId), roomContextPromise, userProfilePromise, ]); const context: PromptContext = { blackboard, conversationHistory, roomContext, summary, userMessage: this.buildPromptUserMessage(request, derived), userProfile, }; const builder = PromptBuilder.forDerived(derived, session, context); const promptVariantId = getPromptVariantId(); if (!promptVariantId) { return builder.build(); } const manifest = loadPromptVariantManifest(promptVariantId); if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) { return builder.build(); } return builder.applyVariant(manifest).build(); } private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string { switch (request.trigger) { case 'nickname-invalid': return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`; case 'photo-failed': return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.'; case 'progress-update': return '현재 새내기 등록 진행상황만 짧게 알려줘.'; case 'timeout-warning': return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.'; case 'completed': return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.'; case 'user-message': if (derived.intent.kind === 'status-question') { return '완료된 것은 빼고 남은 절차만 짧게 말해줘.'; } return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.'; default: return '지금 필요한 온보딩 안내만 짧게 도와줘.'; } } private async loadConversationSummary( chatroomId: string, userId: string, ): Promise { const key = getConversationSummaryKey(chatroomId, userId); return (await this.garnetClient.getString(key)) ?? undefined; } private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise { const now = new Date().toISOString(); const userMessage = request.userMessage?.trim(); await this.blackboardStore.update(request.chatroomId, request.userId, (current) => { const sameIntentStreak = current.lastIntent === derived.intent.kind ? (current.sameIntentStreak ?? 1) + 1 : 1; const openQuestions = userMessage && (derived.intent.kind === 'help-how' || derived.intent.kind === 'help-where' || derived.intent.kind === 'nickname-question' || derived.intent.kind === 'photo-question' || derived.intent.kind === 'discord-question' || derived.intent.kind === 'discord-verify-question') ? [...current.openQuestions, userMessage].slice(-3) : current.openQuestions; const recentNicknameAttempts = request.trigger === 'nickname-invalid' || derived.intent.kind === 'nickname-question' ? Array.from(new Set([...current.recentNicknameAttempts, request.nickname])).slice(-5) : current.recentNicknameAttempts; return { ...current, flaggedIssues: derived.phase.kind === 'nickname-rejected' ? derived.phase.issues.map((issue) => issue.message) : derived.phase.kind === 'completed' ? [] : current.flaggedIssues, lastIntent: derived.intent.kind, lastIntentAt: now, openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions, recentNicknameAttempts, sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak, toneMemory: { ...current.toneMemory, greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt, thankedAt: derived.intent.kind === 'acknowledgement' ? now : current.toneMemory.thankedAt, }, }; }); } private async observeUserMemory( request: AgentRequest, derived: DerivedState, response: AgentResponse, ): Promise { if (!getAgentUserMemoryEnabled()) { return; } await this.userMemoryStore.observeTurn(request.userId, derived, response, { flaggedIssues: derived.phase.kind === 'nickname-rejected' ? derived.phase.issues.map((issue) => issue.message) : [], }); } private makeTemplateFallbackResponse( request: AgentRequest, session: OnboardingSession | null, intent: Intent, derived: DerivedState, ): AgentResponse { switch (request.trigger) { case 'new-member': return this.makeEmptyResponse(request.requestId); case 'nickname-invalid': return this.makeTemplateResponse( request.requestId, getNicknameInvalidMessage(request.nickname, session), ); case 'photo-failed': return this.makeTemplateResponse( request.requestId, getPhotoFailedMessage(request.nickname), ); case 'progress-update': return this.makeTemplateResponse( request.requestId, session ? getProgressMessage(request.nickname, session) : getGenericHelpMessage(request.nickname, session), ); case 'timeout-warning': return this.makeTemplateResponse( request.requestId, getTimeoutWarningMessage(request.nickname), ); case 'completed': return this.makeTemplateResponse(request.requestId, getCompletedMessage(request.nickname)); default: return this.makeTemplateResponse( request.requestId, this.getIntentFallbackText(request.nickname, session, intent, derived), ); } } private getIntentFallbackText( nickname: string, session: OnboardingSession | null, intent: Intent, derived: DerivedState, ): string { switch (intent.kind) { case 'greeting': return getGreetingMessage(); case 'acknowledgement': return getAcknowledgementMessage(); case 'status-question': return getRemainingStatusMessage(session); case 'nickname-question': return derived.phase.kind === 'nickname-rejected' ? getNicknameInvalidMessage(nickname, session) : getNicknameFormatMessage(); case 'photo-question': return derived.phase.kind === 'photo-rejected' ? getPhotoFailedMessage(nickname) : getHeartGuideMessage(); case 'help-where': if ( derived.phase.kind === 'photo-pending' || derived.phase.kind === 'photo-rejected' || (derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo')) ) { return getHeartLocationMessage(); } return getGenericHelpMessage(nickname, session); case 'help-how': if ( derived.phase.kind === 'nickname-pending' || derived.phase.kind === 'nickname-rejected' || (derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('nickname')) ) { return getNicknameFormatMessage(); } if ( derived.phase.kind === 'photo-pending' || derived.phase.kind === 'photo-rejected' || (derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo')) ) { return getHeartGuideMessage(); } return getGenericHelpMessage(nickname, session); case 'discord-question': return getDiscordMessage(nickname, session); case 'discord-verify-question': return getDiscordMessage(nickname, session); case 'age-question': return getAgeRequirementMessage(); case 'chitchat': case 'off-topic-to-other': return ''; case 'unknown': return getGenericHelpMessage(nickname, session); } } private async generateLlmText( messages: ChatMessage[], derived: DerivedState, options?: { allowShortResponse?: boolean; maxTokens?: number; request: AgentRequest; retryOnLowQuality?: boolean; temperature?: number; toolContext?: ToolContext | undefined; tools?: readonly Tool[] | undefined; }, ): Promise { if (!this.llmClient.hasEndpoints()) { return { status: 'failed' }; } const attempts: Array<{ maxTokens?: number; temperature?: number; }> = []; const firstAttempt: { maxTokens?: number; temperature?: number; } = {}; if (options?.maxTokens !== undefined) { firstAttempt.maxTokens = options.maxTokens; } if (options?.temperature !== undefined) { firstAttempt.temperature = options.temperature; } attempts.push(firstAttempt); if (options?.retryOnLowQuality) { const retryAttempt: { maxTokens?: number; temperature?: number; } = { temperature: getLlmTemperature() + 0.1, }; if (options.maxTokens !== undefined) { retryAttempt.maxTokens = options.maxTokens; } attempts.push(retryAttempt); } for (const attempt of attempts) { const requestOptions: { maxTokens?: number; temperature?: number; } = {}; if (attempt.maxTokens !== undefined) { requestOptions.maxTokens = attempt.maxTokens; } if (attempt.temperature !== undefined) { requestOptions.temperature = attempt.temperature; } const toolContext = options?.toolContext ?? {}; const llmText = await this.orchestrator.run(derived, messages, { maxTokens: requestOptions.maxTokens, request: options?.request, temperature: requestOptions.temperature, toolContext, tools: options?.tools, }); if (!llmText) { continue; } const validation = validateOnboardingResponse(llmText, derived, { allowShortResponse: options?.allowShortResponse, }); if (validation.text.includes('[NO_RESPONSE]')) { return { status: 'no_response' }; } if (validation.status === 'reject') { continue; } return { status: 'success', text: validation.text, }; } return { status: 'failed' }; } private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext { return { chatroomId: request.chatroomId, natsClient: this.natsClient, userId: request.userId, session, garnetClient: this.garnetClient, roomContextService: this.roomContextService, toolRegistry: this.toolRegistry, }; } private selectToolsForDerived(derived: DerivedState): readonly Tool[] { const maybeUseToolSearch = ( tools: readonly Tool[], ): readonly Tool[] => { if ( getAgentToolSearchEnabled() && tools.length >= getAgentToolSearchThreshold() ) { return [createToolSearchTool(this.toolRegistry)]; } return tools; }; if (derived.intent.kind === 'discord-verify-question') { return maybeUseToolSearch( this.tools.filter((tool) => tool.name === 'verify-discord-nickname'), ); } if (derived.phase.kind === 'nickname-rejected') { return maybeUseToolSearch( this.tools.filter( (tool) => tool.name === 'validate-nickname' || tool.name === 'lookup-community-rule' || tool.name === 'check-onboarding-progress', ), ); } if (derived.intent.kind === 'status-question') { return maybeUseToolSearch( this.tools.filter((tool) => tool.name === 'check-onboarding-progress'), ); } if (derived.intent.kind === 'discord-question') { return maybeUseToolSearch( this.tools.filter( (tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname', ), ); } return maybeUseToolSearch(this.tools); } private makeTemplateResponse(requestId: string, text: string): AgentResponse { return { requestId, text, confidence: 0.8, source: 'template', }; } private makeLlmResponse(requestId: string, text: string): AgentResponse { return { requestId, text, confidence: 0.9, source: 'llm', }; } private makeEmptyResponse(requestId: string): AgentResponse { return { requestId, text: '', confidence: 0, source: 'template', }; } private getCurrentLatencyMs(): number | undefined { const startedAt = getContext()?.startedAt; const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN; if (!Number.isFinite(startedTimestamp)) { return undefined; } return Math.max(0, Date.now() - startedTimestamp); } }