Files
lolympus-agent/src/skills/onboarding/skill.ts
T

808 lines
26 KiB
TypeScript

import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
import { getConversationSummaryKey } from '@conv/conversation-store';
import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats';
import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { ChatMessage } from '@llm/types';
import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { meetsTrustTier } from '../../eval/governance/trust.js';
import { createBuiltInToolRegistry } from '@tools/builtins/index';
import type { ToolRegistry } from '@tools/registry';
import type { Tool, ToolContext } from '@tools/tool';
import { createToolSearchTool } from '@tools/tool-search';
import {
loadPromptVariantManifest,
PromptBuilder,
} from './prompts/builder.js';
import {
getAcknowledgementMessage,
getAgeRequirementMessage,
getCompletedMessage,
getDiscordMessage,
getGreetingMessage,
getGenericHelpMessage,
getHeartGuideMessage,
getHeartLocationMessage,
getNicknameInvalidMessage,
getNicknameFormatMessage,
getOnboardingTemplate,
getPhotoFailedMessage,
getProgressMessage,
getRemainingStatusMessage,
getTimeoutWarningMessage,
} from './onboarding-templates.js';
import {
getAgentAbstentionClarifyThreshold,
getAgentAbstentionEnabled,
getAgentAbstentionThreshold,
getAgentRouterMode,
getAgentRouterWeightsPath,
getAgentTrustMinTier,
getAgentToolSearchEnabled,
getAgentToolSearchThreshold,
getAgentUserMemoryEnabled,
getOnboardingSessionKeyPrefix,
getLlmTemperature,
getPromptVariantId,
} from '../../config.js';
import { OnboardingBlackboardStore } from './state/blackboard.js';
import { UserMemoryStore } from './state/user-memory.js';
import { deriveState } from './state/derive.js';
import { OnboardingIntentClassifier } from './state/classifier.js';
import type {
ClassifiedIntent,
DerivedState,
Intent,
} from './state/onboarding-state.js';
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
import {
decideEarlyAbstention,
type AbstentionDecision,
} from './runtime/abstention.js';
import {
decideHybridRouting,
decidePreGenerationRouting,
loadLearnedRouterModel,
} from './runtime/router.js';
import {
getActiveProcessTrace,
ProcessTraceCollector,
runWithProcessTrace,
} from './runtime/trace.js';
import { validateOnboardingResponse } from './runtime/validator.js';
import { normalizeOnboardingSession } from './session.js';
import type {
Blackboard,
OnboardingSession,
PromptContext,
StoredOnboardingSession,
} from './types.js';
type LlmTextResult =
| { status: 'success'; text: string }
| { status: 'no_response' }
| { status: 'failed' };
export class OnboardingSkill implements Skill {
readonly name = 'onboarding';
private readonly classifier: OnboardingIntentClassifier;
private readonly blackboardStore: OnboardingBlackboardStore;
private readonly orchestrator: OnboardingOrchestrator;
private readonly shadowEvalPublisher: ShadowEvalPublisher;
private readonly toolRegistry: ToolRegistry;
private readonly tools: readonly Tool<unknown, unknown>[];
private readonly userMemoryStore: UserMemoryStore;
constructor(
private readonly garnetClient: GarnetClient,
private readonly llmClient: LlmClient,
private readonly roomContextService?: RoomContextService,
shadowEvalPublisher?: ShadowEvalPublisher,
private readonly natsClient?: Pick<NatsClient, 'request'>,
) {
this.classifier = new OnboardingIntentClassifier(llmClient);
this.blackboardStore = new OnboardingBlackboardStore(garnetClient);
this.userMemoryStore = new UserMemoryStore(garnetClient);
this.orchestrator = new OnboardingOrchestrator(llmClient);
this.shadowEvalPublisher = shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
this.toolRegistry = createBuiltInToolRegistry();
this.tools = this.toolRegistry.list();
}
canHandle(request: AgentRequest): boolean {
return request.skill === 'onboarding';
}
async handle(
request: AgentRequest,
conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
if (getActiveProcessTrace()) {
return this.handleWithTrace(request, conversationHistory);
}
const collector = new ProcessTraceCollector();
return runWithProcessTrace(collector, () =>
this.handleWithTrace(request, conversationHistory),
);
}
private async handleWithTrace(
request: AgentRequest,
conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const logger = getRequestLogger();
const trace = getActiveProcessTrace();
const session = await this.loadSession(request.chatroomId, request.userId);
if (request.trigger === 'user-message' && !request.userMessage?.trim()) {
return this.makeEmptyResponse(request.requestId);
}
const classifyStep = trace?.startStep('classify');
const classified = await this.classifier.classify(request.userMessage, request.trigger);
let classifyFinished = false;
const finishClassify = (abstention?: AbstentionDecision): void => {
if (classifyFinished) {
return;
}
classifyFinished = true;
classifyStep?.finish({
...(abstention ? { abstention } : {}),
confidence: classified.confidence,
intent: classified.intent.kind,
source: classified.source,
});
};
const deriveStep = trace?.startStep('derive');
const derived = deriveState(session, request.trigger, classified.intent);
const routerMode = getAgentRouterMode();
if (routerMode === 'static' || routerMode === 'learned-v1') {
const routerInput = {
classified,
derived,
request,
session,
};
derived.routingDecision =
routerMode === 'learned-v1'
? decideHybridRouting(routerInput, {
learnedModel: loadLearnedRouterModel(getAgentRouterWeightsPath()),
})
: decidePreGenerationRouting(routerInput);
}
deriveStep?.finish({
phase: derived.phase.kind,
responseMode: derived.responseMode,
routingDecision: derived.routingDecision,
shouldRespond: derived.shouldRespond,
tier: derived.tier,
});
if (getAgentAbstentionEnabled()) {
const decision = await this.decideEarlyAbstention(request, classified, derived);
derived.abstentionScore = decision.score;
finishClassify(decision);
if (decision.action.kind === 'no-response') {
const response = this.makeEmptyResponse(request.requestId);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse('[NO_RESPONSE]', derived, {
allowShortResponse: true,
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
});
return response;
}
if (decision.action.kind === 'clarify') {
const response = this.makeTemplateResponse(
request.requestId,
getOnboardingTemplate(decision.action.templateKey),
);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse(response.text, derived, {
allowShortResponse: true,
});
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
});
await this.observeUserMemory(request, derived, response).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
});
return response;
}
} else {
finishClassify();
}
if (!derived.shouldRespond) {
const response = this.makeEmptyResponse(request.requestId);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse('[NO_RESPONSE]', derived, {
allowShortResponse: true,
});
return response;
}
let response: AgentResponse;
if (derived.responseMode === 'template') {
response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived);
} else {
const llmResult = await this.handleLlm(request, session, conversationHistory, derived);
if (llmResult.status === 'success') {
response = this.makeLlmResponse(request.requestId, llmResult.text);
} else if (llmResult.status === 'no_response') {
response = this.makeEmptyResponse(request.requestId);
} else {
response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived);
}
}
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, {
allowShortResponse: derived.allowShortResponse,
});
if (derived.intent.kind === 'discord-verify-question') {
logger.debug(
{
responseSource: response.source,
responseText: response.text,
},
'[OnboardingSkill] Final discord verification response',
);
}
if (response.text.trim()) {
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
});
await this.observeUserMemory(request, derived, response).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
});
}
return response;
}
private async loadSession(chatroomId: string, userId: string): Promise<OnboardingSession | null> {
const prefix = getOnboardingSessionKeyPrefix();
const key = `${prefix}:onboarding:${chatroomId}:${userId}`;
const rawSession = await this.garnetClient.getJson<StoredOnboardingSession>(key);
return normalizeOnboardingSession(rawSession);
}
private async decideEarlyAbstention(
request: AgentRequest,
classified: ClassifiedIntent,
derived: DerivedState,
): Promise<AbstentionDecision> {
const logger = getRequestLogger();
let blackboard: Blackboard | null = null;
try {
blackboard = await this.blackboardStore.load(request.chatroomId, request.userId);
} catch (error) {
logger.warn({ err: error }, '[OnboardingSkill] Failed to load blackboard for abstention');
}
return decideEarlyAbstention({
blackboard,
classified,
derived,
request,
thresholds: {
abstain: getAgentAbstentionThreshold(),
clarify: getAgentAbstentionClarifyThreshold(),
},
});
}
private async handleLlm(
request: AgentRequest,
session: OnboardingSession | null,
conversationHistory: ConversationMessage[],
derived: DerivedState,
): Promise<LlmTextResult> {
const messages = await this.buildLlmMessages(request, session, conversationHistory, derived);
return this.generateLlmText(messages, derived, {
allowShortResponse: derived.allowShortResponse,
maxTokens: derived.suggestedMaxTokens,
request,
retryOnLowQuality:
request.trigger === 'user-message' &&
!derived.allowShortResponse &&
derived.responseMode !== 'llm-with-tools',
toolContext: this.buildToolContext(request, session),
tools: this.selectToolsForDerived(derived),
});
}
private async buildLlmMessages(
request: AgentRequest,
session: OnboardingSession | null,
conversationHistory: ConversationMessage[],
derived: DerivedState,
): Promise<ChatMessage[]> {
const logger = getRequestLogger();
const roomContextPromise =
request.trigger === 'user-message' && this.roomContextService
? this.roomContextService.getRecentMessages(request.chatroomId).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to fetch room context');
return undefined;
})
: Promise.resolve(undefined);
const userProfilePromise = getAgentUserMemoryEnabled()
? this.userMemoryStore.load(request.userId).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory');
return null;
})
: Promise.resolve(null);
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
this.blackboardStore.load(request.chatroomId, request.userId),
this.loadConversationSummary(request.chatroomId, request.userId),
roomContextPromise,
userProfilePromise,
]);
const context: PromptContext = {
blackboard,
conversationHistory,
roomContext,
summary,
userMessage: this.buildPromptUserMessage(request, derived),
userProfile,
};
const builder = PromptBuilder.forDerived(derived, session, context);
const promptVariantId = getPromptVariantId();
if (!promptVariantId) {
return builder.build();
}
const manifest = loadPromptVariantManifest(promptVariantId);
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
return builder.build();
}
return builder.applyVariant(manifest).build();
}
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
switch (request.trigger) {
case 'nickname-invalid':
return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`;
case 'photo-failed':
return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.';
case 'progress-update':
return '현재 새내기 등록 진행상황만 짧게 알려줘.';
case 'timeout-warning':
return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.';
case 'completed':
return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.';
case 'user-message':
if (derived.intent.kind === 'status-question') {
return '완료된 것은 빼고 남은 절차만 짧게 말해줘.';
}
return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.';
default:
return '지금 필요한 온보딩 안내만 짧게 도와줘.';
}
}
private async loadConversationSummary(
chatroomId: string,
userId: string,
): Promise<string | undefined> {
const key = getConversationSummaryKey(chatroomId, userId);
return (await this.garnetClient.getString(key)) ?? undefined;
}
private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise<void> {
const now = new Date().toISOString();
const userMessage = request.userMessage?.trim();
await this.blackboardStore.update(request.chatroomId, request.userId, (current) => {
const sameIntentStreak =
current.lastIntent === derived.intent.kind
? (current.sameIntentStreak ?? 1) + 1
: 1;
const openQuestions =
userMessage &&
(derived.intent.kind === 'help-how' ||
derived.intent.kind === 'help-where' ||
derived.intent.kind === 'nickname-question' ||
derived.intent.kind === 'photo-question' ||
derived.intent.kind === 'discord-question' ||
derived.intent.kind === 'discord-verify-question')
? [...current.openQuestions, userMessage].slice(-3)
: current.openQuestions;
const recentNicknameAttempts =
request.trigger === 'nickname-invalid' || derived.intent.kind === 'nickname-question'
? Array.from(new Set([...current.recentNicknameAttempts, request.nickname])).slice(-5)
: current.recentNicknameAttempts;
return {
...current,
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: derived.phase.kind === 'completed'
? []
: current.flaggedIssues,
lastIntent: derived.intent.kind,
lastIntentAt: now,
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
recentNicknameAttempts,
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
toneMemory: {
...current.toneMemory,
greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt,
thankedAt: derived.intent.kind === 'acknowledgement' ? now : current.toneMemory.thankedAt,
},
};
});
}
private async observeUserMemory(
request: AgentRequest,
derived: DerivedState,
response: AgentResponse,
): Promise<void> {
if (!getAgentUserMemoryEnabled()) {
return;
}
await this.userMemoryStore.observeTurn(request.userId, derived, response, {
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: [],
});
}
private makeTemplateFallbackResponse(
request: AgentRequest,
session: OnboardingSession | null,
intent: Intent,
derived: DerivedState,
): AgentResponse {
switch (request.trigger) {
case 'new-member':
return this.makeEmptyResponse(request.requestId);
case 'nickname-invalid':
return this.makeTemplateResponse(
request.requestId,
getNicknameInvalidMessage(request.nickname, session),
);
case 'photo-failed':
return this.makeTemplateResponse(
request.requestId,
getPhotoFailedMessage(request.nickname),
);
case 'progress-update':
return this.makeTemplateResponse(
request.requestId,
session
? getProgressMessage(request.nickname, session)
: getGenericHelpMessage(request.nickname, session),
);
case 'timeout-warning':
return this.makeTemplateResponse(
request.requestId,
getTimeoutWarningMessage(request.nickname),
);
case 'completed':
return this.makeTemplateResponse(request.requestId, getCompletedMessage(request.nickname));
default:
return this.makeTemplateResponse(
request.requestId,
this.getIntentFallbackText(request.nickname, session, intent, derived),
);
}
}
private getIntentFallbackText(
nickname: string,
session: OnboardingSession | null,
intent: Intent,
derived: DerivedState,
): string {
switch (intent.kind) {
case 'greeting':
return getGreetingMessage();
case 'acknowledgement':
return getAcknowledgementMessage();
case 'status-question':
return getRemainingStatusMessage(session);
case 'nickname-question':
return derived.phase.kind === 'nickname-rejected'
? getNicknameInvalidMessage(nickname, session)
: getNicknameFormatMessage();
case 'photo-question':
return derived.phase.kind === 'photo-rejected'
? getPhotoFailedMessage(nickname)
: getHeartGuideMessage();
case 'help-where':
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
) {
return getHeartLocationMessage();
}
return getGenericHelpMessage(nickname, session);
case 'help-how':
if (
derived.phase.kind === 'nickname-pending' ||
derived.phase.kind === 'nickname-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('nickname'))
) {
return getNicknameFormatMessage();
}
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
) {
return getHeartGuideMessage();
}
return getGenericHelpMessage(nickname, session);
case 'discord-question':
return getDiscordMessage(nickname, session);
case 'discord-verify-question':
return getDiscordMessage(nickname, session);
case 'age-question':
return getAgeRequirementMessage();
case 'chitchat':
case 'off-topic-to-other':
return '';
case 'unknown':
return getGenericHelpMessage(nickname, session);
}
}
private async generateLlmText(
messages: ChatMessage[],
derived: DerivedState,
options?: {
allowShortResponse?: boolean;
maxTokens?: number;
request: AgentRequest;
retryOnLowQuality?: boolean;
temperature?: number;
toolContext?: ToolContext | undefined;
tools?: readonly Tool<unknown, unknown>[] | undefined;
},
): Promise<LlmTextResult> {
if (!this.llmClient.hasEndpoints()) {
return { status: 'failed' };
}
const attempts: Array<{
maxTokens?: number;
temperature?: number;
}> = [];
const firstAttempt: {
maxTokens?: number;
temperature?: number;
} = {};
if (options?.maxTokens !== undefined) {
firstAttempt.maxTokens = options.maxTokens;
}
if (options?.temperature !== undefined) {
firstAttempt.temperature = options.temperature;
}
attempts.push(firstAttempt);
if (options?.retryOnLowQuality) {
const retryAttempt: {
maxTokens?: number;
temperature?: number;
} = {
temperature: getLlmTemperature() + 0.1,
};
if (options.maxTokens !== undefined) {
retryAttempt.maxTokens = options.maxTokens;
}
attempts.push(retryAttempt);
}
for (const attempt of attempts) {
const requestOptions: {
maxTokens?: number;
temperature?: number;
} = {};
if (attempt.maxTokens !== undefined) {
requestOptions.maxTokens = attempt.maxTokens;
}
if (attempt.temperature !== undefined) {
requestOptions.temperature = attempt.temperature;
}
const toolContext = options?.toolContext ?? {};
const llmText = await this.orchestrator.run(derived, messages, {
maxTokens: requestOptions.maxTokens,
request: options?.request,
temperature: requestOptions.temperature,
toolContext,
tools: options?.tools,
});
if (!llmText) {
continue;
}
const validation = validateOnboardingResponse(llmText, derived, {
allowShortResponse: options?.allowShortResponse,
});
if (validation.text.includes('[NO_RESPONSE]')) {
return { status: 'no_response' };
}
if (validation.status === 'reject') {
continue;
}
return {
status: 'success',
text: validation.text,
};
}
return { status: 'failed' };
}
private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext {
return {
chatroomId: request.chatroomId,
natsClient: this.natsClient,
userId: request.userId,
session,
garnetClient: this.garnetClient,
roomContextService: this.roomContextService,
toolRegistry: this.toolRegistry,
};
}
private selectToolsForDerived(derived: DerivedState): readonly Tool<unknown, unknown>[] {
const maybeUseToolSearch = (
tools: readonly Tool<unknown, unknown>[],
): readonly Tool<unknown, unknown>[] => {
if (
getAgentToolSearchEnabled() &&
tools.length >= getAgentToolSearchThreshold()
) {
return [createToolSearchTool(this.toolRegistry)];
}
return tools;
};
if (derived.intent.kind === 'discord-verify-question') {
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'verify-discord-nickname'),
);
}
if (derived.phase.kind === 'nickname-rejected') {
return maybeUseToolSearch(
this.tools.filter(
(tool) =>
tool.name === 'validate-nickname' ||
tool.name === 'lookup-community-rule' ||
tool.name === 'check-onboarding-progress',
),
);
}
if (derived.intent.kind === 'status-question') {
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'check-onboarding-progress'),
);
}
if (derived.intent.kind === 'discord-question') {
return maybeUseToolSearch(
this.tools.filter(
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
),
);
}
return maybeUseToolSearch(this.tools);
}
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
return {
requestId,
text,
confidence: 0.8,
source: 'template',
};
}
private makeLlmResponse(requestId: string, text: string): AgentResponse {
return {
requestId,
text,
confidence: 0.9,
source: 'llm',
};
}
private makeEmptyResponse(requestId: string): AgentResponse {
return {
requestId,
text: '',
confidence: 0,
source: 'template',
};
}
private getCurrentLatencyMs(): number | undefined {
const startedAt = getContext()?.startedAt;
const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN;
if (!Number.isFinite(startedTimestamp)) {
return undefined;
}
return Math.max(0, Date.now() - startedTimestamp);
}
}