Files
lolympus-agent/src/skills/onboarding/skill.ts
T
dahyeon 21df14cd00 Add cost-aware routing and LLM budget tracking
- BudgetTracker tracks LLM token cost separately from tool cost
- LlmClient auto-records cost into request-scoped budget
- Router selects tier from classify confidence, intent risk, escalation count
- Supervisor short-circuits via regex fast-path when rule confidence is high
- Onboarding skill loads user escalation history to inform routing
2026-05-19 01:28:54 -07:00

590 lines
18 KiB
TypeScript

import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats';
import type { EscalationHistoryService } from '../../memory/escalation-history.js';
import type { MemoryService } from '../../memory/memory-types.js';
import {
buildLongTermSummary,
shouldRefreshLongTermSummary,
} from '../../memory/long-term-summarizer.js';
import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import type { SkillCapabilityCard } from '@skills/skill-loader';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { createOnboardingToolRegistry } from './tools.js';
import type { ToolRegistry } from '@tools/registry';
import type { Tool, ToolContext } from '@tools/tool';
import {
getCompletedMessage,
getGenericHelpMessage,
getNicknameInvalidMessage,
getOnboardingTemplate,
getPhotoFailedMessage,
getProgressMessage,
getTimeoutWarningMessage,
} from './onboarding-templates.js';
import {
getAgentAbstentionClarifyThreshold,
getAgentAbstentionEnabled,
getAgentAbstentionThreshold,
getAgentRouterMode,
getAgentRouterWeightsPath,
getAgentUserMemoryEnabled,
getOnboardingSessionKeyPrefix,
} from '../../config.js';
import { OnboardingBlackboardStore } from './state/blackboard.js';
import { UserMemoryStore } from './state/user-memory.js';
import { deriveState } from './state/derive.js';
import { OnboardingIntentClassifier } from './state/classifier.js';
import type {
ClassifiedIntent,
DerivedState,
Intent,
} from './state/onboarding-state.js';
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
import {
decideEarlyAbstention,
type AbstentionDecision,
} from './runtime/abstention.js';
import {
decideHybridRouting,
decidePreGenerationRouting,
loadLearnedRouterModel,
} from './runtime/router.js';
import {
getActiveProcessTrace,
ProcessTraceCollector,
runWithProcessTrace,
} from './runtime/trace.js';
import { validateOnboardingResponse } from './runtime/validator.js';
import { getIntentFallbackText } from './runtime/fallback-text-resolver.js';
import { buildLlmMessages } from './runtime/message-builder.js';
import {
generateLlmText,
type LlmTextResult,
} from './runtime/text-generator.js';
import { selectToolsForDerived } from './runtime/tool-selector.js';
import { rememberTurn } from './runtime/state-updater.js';
import { normalizeOnboardingSession } from './session.js';
import type {
Blackboard,
OnboardingSession,
StoredOnboardingSession,
} from './types.js';
const CAPABILITY_CARD: SkillCapabilityCard = {
canHandle: [
'new-member',
'nickname-invalid',
'photo-failed',
'progress-update',
'timeout-warning',
'completed',
'onboarding-help',
],
cannotHandle: [
'moderation-report',
'sanctions',
'appeal',
'policy-enforcement',
],
domain: 'onboarding',
handoffTargets: ['moderation', 'supervisor'],
outputContract: {
format: 'text',
maxChars: '500',
requiredFields: 'text',
},
riskTier: 'medium',
};
export class OnboardingSkill implements Skill {
readonly name = 'onboarding';
readonly priority = 70;
private readonly classifier: OnboardingIntentClassifier;
private readonly blackboardStore: OnboardingBlackboardStore;
private readonly orchestrator: OnboardingOrchestrator;
private readonly shadowEvalPublisher: ShadowEvalPublisher;
private readonly toolRegistry: ToolRegistry;
private readonly tools: readonly Tool<unknown, unknown>[];
private readonly userMemoryStore: UserMemoryStore;
constructor(
private readonly garnetClient: GarnetClient,
private readonly llmClient: LlmClient,
private readonly roomContextService?: RoomContextService,
shadowEvalPublisher?: ShadowEvalPublisher,
private readonly natsClient?: Pick<NatsClient, 'request'>,
private readonly memoryService?: MemoryService,
private readonly escalationHistoryService?: EscalationHistoryService,
) {
this.classifier = new OnboardingIntentClassifier(llmClient);
this.blackboardStore = new OnboardingBlackboardStore(
garnetClient,
memoryService?.session,
);
this.userMemoryStore = new UserMemoryStore(
garnetClient,
memoryService?.longTerm,
);
this.orchestrator = new OnboardingOrchestrator(llmClient);
this.shadowEvalPublisher = shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
this.toolRegistry = createOnboardingToolRegistry();
this.tools = this.toolRegistry.list();
}
canHandle(request: AgentRequest): boolean {
return request.skill === 'onboarding';
}
capabilityCard(): SkillCapabilityCard {
return CAPABILITY_CARD;
}
async handle(
request: AgentRequest,
conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
if (getActiveProcessTrace()) {
return this.handleWithTrace(request, conversationHistory);
}
const collector = new ProcessTraceCollector();
return runWithProcessTrace(collector, () =>
this.handleWithTrace(request, conversationHistory),
);
}
private async handleWithTrace(
request: AgentRequest,
conversationHistory: ConversationMessage[],
): Promise<AgentResponse> {
const logger = getRequestLogger();
const trace = getActiveProcessTrace();
const session = await this.loadSession(request.chatroomId, request.userId);
if (request.trigger === 'user-message' && !request.userMessage?.trim()) {
return this.makeEmptyResponse(request.requestId);
}
const classifyStep = trace?.startStep('classify');
const classified = await this.classifier.classify(request.userMessage, request.trigger);
let classifyFinished = false;
const finishClassify = (abstention?: AbstentionDecision): void => {
if (classifyFinished) {
return;
}
classifyFinished = true;
classifyStep?.finish({
...(abstention ? { abstention } : {}),
confidence: classified.confidence,
intent: classified.intent.kind,
source: classified.source,
});
};
const deriveStep = trace?.startStep('derive');
const derived = deriveState(session, request.trigger, classified.intent);
const escalationHistory = await this.loadEscalationHistory(request.userId);
const routerMode = getAgentRouterMode();
if (routerMode === 'static' || routerMode === 'learned-v1') {
const routerInput = {
classified,
derived,
intentRiskTier: this.capabilityCard().riskTier,
recentLatencyP95Ms: this.getRecentLatencyP95Ms(request),
request,
session,
userEscalationCount: escalationHistory?.totalEscalations ?? 0,
};
derived.routingDecision =
routerMode === 'learned-v1'
? decideHybridRouting(routerInput, {
learnedModel: loadLearnedRouterModel(getAgentRouterWeightsPath()),
})
: decidePreGenerationRouting(routerInput);
}
deriveStep?.finish({
phase: derived.phase.kind,
responseMode: derived.responseMode,
routingDecision: derived.routingDecision,
shouldRespond: derived.shouldRespond,
tier: derived.tier,
});
if (getAgentAbstentionEnabled()) {
const decision = await this.decideEarlyAbstention(request, classified, derived);
derived.abstentionScore = decision.score;
finishClassify(decision);
if (decision.action.kind === 'no-response') {
const response = this.makeEmptyResponse(request.requestId);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse('[NO_RESPONSE]', derived, {
allowShortResponse: true,
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
return response;
}
if (decision.action.kind === 'clarify') {
const response = this.makeTemplateResponse(
request.requestId,
getOnboardingTemplate(decision.action.templateKey),
);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse(response.text, derived, {
allowShortResponse: true,
});
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
});
await this.observeUserMemory(request, derived, response).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
return response;
}
} else {
finishClassify();
}
if (!derived.shouldRespond) {
const response = this.makeEmptyResponse(request.requestId);
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse('[NO_RESPONSE]', derived, {
allowShortResponse: true,
});
return response;
}
let response: AgentResponse;
if (derived.responseMode === 'template') {
response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived);
} else {
const llmResult = await this.handleLlm(request, session, conversationHistory, derived);
if (llmResult.status === 'success') {
response = this.makeLlmResponse(request.requestId, llmResult.text);
} else if (llmResult.status === 'no_response') {
response = this.makeEmptyResponse(request.requestId);
} else {
response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived);
}
}
trace?.startStep('respond').finish({
draftText: response.text,
tokensIn: 0,
tokensOut: 0,
});
validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, {
allowShortResponse: derived.allowShortResponse,
});
if (derived.intent.kind === 'discord-verify-question') {
logger.debug(
{
responseSource: response.source,
responseText: response.text,
},
'[OnboardingSkill] Final discord verification response',
);
}
if (response.text.trim()) {
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
});
await this.observeUserMemory(request, derived, response).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
request,
response,
skill: this.name,
});
}
return response;
}
private async loadSession(chatroomId: string, userId: string): Promise<OnboardingSession | null> {
const prefix = getOnboardingSessionKeyPrefix();
const key = `${prefix}:onboarding:${chatroomId}:${userId}`;
const rawSession = await this.garnetClient.getJson<StoredOnboardingSession>(key);
return normalizeOnboardingSession(rawSession);
}
private async loadEscalationHistory(userId: string) {
if (!this.escalationHistoryService) {
return null;
}
try {
return await this.escalationHistoryService.load(userId);
} catch (error) {
getRequestLogger().warn(
{ err: error },
'[OnboardingSkill] Failed to load escalation history',
);
return null;
}
}
private getRecentLatencyP95Ms(request: AgentRequest): number | undefined {
const raw = request.context?.['recentLatencyP95Ms'];
return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined;
}
private async decideEarlyAbstention(
request: AgentRequest,
classified: ClassifiedIntent,
derived: DerivedState,
): Promise<AbstentionDecision> {
const logger = getRequestLogger();
let blackboard: Blackboard | null = null;
try {
blackboard = await this.blackboardStore.load(request.chatroomId, request.userId);
} catch (error) {
logger.warn({ err: error }, '[OnboardingSkill] Failed to load blackboard for abstention');
}
return decideEarlyAbstention({
blackboard,
classified,
derived,
request,
thresholds: {
abstain: getAgentAbstentionThreshold(),
clarify: getAgentAbstentionClarifyThreshold(),
},
});
}
private async handleLlm(
request: AgentRequest,
session: OnboardingSession | null,
conversationHistory: ConversationMessage[],
derived: DerivedState,
): Promise<LlmTextResult> {
if (!this.llmClient.hasEndpoints()) {
return { status: 'failed' };
}
const messages = await buildLlmMessages(
{
blackboardStore: this.blackboardStore,
garnetClient: this.garnetClient,
memoryService: this.memoryService,
roomContextService: this.roomContextService,
userMemoryStore: this.userMemoryStore,
},
{
conversationHistory,
derived,
request,
session,
},
);
return generateLlmText(this.orchestrator, messages, derived, {
allowShortResponse: derived.allowShortResponse,
maxTokens: derived.suggestedMaxTokens,
request,
retryOnLowQuality:
request.trigger === 'user-message' &&
!derived.allowShortResponse &&
derived.responseMode !== 'llm-with-tools',
toolContext: this.buildToolContext(request, session),
tools: selectToolsForDerived(derived, this.tools, this.toolRegistry),
});
}
private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise<void> {
return rememberTurn(this.blackboardStore, request, derived);
}
private async observeUserMemory(
request: AgentRequest,
derived: DerivedState,
response: AgentResponse,
): Promise<void> {
if (!getAgentUserMemoryEnabled()) {
return;
}
const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, {
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: [],
});
if (!shouldRefreshLongTermSummary(profile)) {
return;
}
void this.refreshLongTermSummary(request, profile);
}
private async refreshLongTermSummary(
request: AgentRequest,
profile: Awaited<ReturnType<UserMemoryStore['observeTurn']>>,
): Promise<void> {
try {
const conversationSummary =
(await this.memoryService?.session.getSummary(
request.chatroomId,
request.userId,
)) ?? null;
const longTermSummary = await buildLongTermSummary(
this.llmClient,
profile,
conversationSummary,
);
await this.userMemoryStore.updateProfile(request.userId, {
longTermSummary,
});
} catch (error) {
getRequestLogger().warn(
{ err: error },
'[OnboardingSkill] Failed to refresh long-term user summary',
);
}
}
private makeTemplateFallbackResponse(
request: AgentRequest,
session: OnboardingSession | null,
intent: Intent,
derived: DerivedState,
): AgentResponse {
switch (request.trigger) {
case 'new-member':
return this.makeEmptyResponse(request.requestId);
case 'nickname-invalid':
return this.makeTemplateResponse(
request.requestId,
getNicknameInvalidMessage(request.nickname, session),
);
case 'photo-failed':
return this.makeTemplateResponse(
request.requestId,
getPhotoFailedMessage(request.nickname),
);
case 'progress-update':
return this.makeTemplateResponse(
request.requestId,
session
? getProgressMessage(request.nickname, session)
: getGenericHelpMessage(request.nickname, session),
);
case 'timeout-warning':
return this.makeTemplateResponse(
request.requestId,
getTimeoutWarningMessage(request.nickname),
);
case 'completed':
return this.makeTemplateResponse(request.requestId, getCompletedMessage(request.nickname));
default:
return this.makeTemplateResponse(
request.requestId,
getIntentFallbackText(request.nickname, session, intent, derived),
);
}
}
private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext {
return {
chatroomId: request.chatroomId,
natsClient: this.natsClient,
userId: request.userId,
session,
garnetClient: this.garnetClient,
memoryService: this.memoryService,
roomContextService: this.roomContextService,
toolRegistry: this.toolRegistry,
};
}
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
return {
requestId,
text,
confidence: 0.8,
source: 'template',
};
}
private makeLlmResponse(requestId: string, text: string): AgentResponse {
return {
requestId,
text,
confidence: 0.9,
source: 'llm',
};
}
private makeEmptyResponse(requestId: string): AgentResponse {
return {
requestId,
text: '',
confidence: 0,
source: 'template',
};
}
private getCurrentLatencyMs(): number | undefined {
const startedAt = getContext()?.startedAt;
const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN;
if (!Number.isFinite(startedTimestamp)) {
return undefined;
}
return Math.max(0, Date.now() - startedTimestamp);
}
}