- BudgetTracker tracks LLM token cost separately from tool cost - LlmClient auto-records cost into request-scoped budget - Router selects tier from classify confidence, intent risk, escalation count - Supervisor short-circuits via regex fast-path when rule confidence is high - Onboarding skill loads user escalation history to inform routing
590 lines
18 KiB
TypeScript
590 lines
18 KiB
TypeScript
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
|
|
import type { GarnetClient } from '@common/garnet';
|
|
import type { NatsClient } from '@common/nats';
|
|
import type { EscalationHistoryService } from '../../memory/escalation-history.js';
|
|
import type { MemoryService } from '../../memory/memory-types.js';
|
|
import {
|
|
buildLongTermSummary,
|
|
shouldRefreshLongTermSummary,
|
|
} from '../../memory/long-term-summarizer.js';
|
|
import { getContext, getRequestLogger } from '@obs/context';
|
|
import type { LlmClient } from '@llm/llm-client';
|
|
import type { RoomContextService } from '@room/room-context-service';
|
|
import type { Skill } from '@skills/skill';
|
|
import type { SkillCapabilityCard } from '@skills/skill-loader';
|
|
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
|
|
import { anonymizeProcessTrace } from '@eval/anonymize';
|
|
import { createOnboardingToolRegistry } from './tools.js';
|
|
import type { ToolRegistry } from '@tools/registry';
|
|
import type { Tool, ToolContext } from '@tools/tool';
|
|
import {
|
|
getCompletedMessage,
|
|
getGenericHelpMessage,
|
|
getNicknameInvalidMessage,
|
|
getOnboardingTemplate,
|
|
getPhotoFailedMessage,
|
|
getProgressMessage,
|
|
getTimeoutWarningMessage,
|
|
} from './onboarding-templates.js';
|
|
import {
|
|
getAgentAbstentionClarifyThreshold,
|
|
getAgentAbstentionEnabled,
|
|
getAgentAbstentionThreshold,
|
|
getAgentRouterMode,
|
|
getAgentRouterWeightsPath,
|
|
getAgentUserMemoryEnabled,
|
|
getOnboardingSessionKeyPrefix,
|
|
} from '../../config.js';
|
|
import { OnboardingBlackboardStore } from './state/blackboard.js';
|
|
import { UserMemoryStore } from './state/user-memory.js';
|
|
import { deriveState } from './state/derive.js';
|
|
import { OnboardingIntentClassifier } from './state/classifier.js';
|
|
import type {
|
|
ClassifiedIntent,
|
|
DerivedState,
|
|
Intent,
|
|
} from './state/onboarding-state.js';
|
|
import { OnboardingOrchestrator } from './runtime/orchestrator.js';
|
|
import {
|
|
decideEarlyAbstention,
|
|
type AbstentionDecision,
|
|
} from './runtime/abstention.js';
|
|
import {
|
|
decideHybridRouting,
|
|
decidePreGenerationRouting,
|
|
loadLearnedRouterModel,
|
|
} from './runtime/router.js';
|
|
import {
|
|
getActiveProcessTrace,
|
|
ProcessTraceCollector,
|
|
runWithProcessTrace,
|
|
} from './runtime/trace.js';
|
|
import { validateOnboardingResponse } from './runtime/validator.js';
|
|
import { getIntentFallbackText } from './runtime/fallback-text-resolver.js';
|
|
import { buildLlmMessages } from './runtime/message-builder.js';
|
|
import {
|
|
generateLlmText,
|
|
type LlmTextResult,
|
|
} from './runtime/text-generator.js';
|
|
import { selectToolsForDerived } from './runtime/tool-selector.js';
|
|
import { rememberTurn } from './runtime/state-updater.js';
|
|
import { normalizeOnboardingSession } from './session.js';
|
|
import type {
|
|
Blackboard,
|
|
OnboardingSession,
|
|
StoredOnboardingSession,
|
|
} from './types.js';
|
|
|
|
const CAPABILITY_CARD: SkillCapabilityCard = {
|
|
canHandle: [
|
|
'new-member',
|
|
'nickname-invalid',
|
|
'photo-failed',
|
|
'progress-update',
|
|
'timeout-warning',
|
|
'completed',
|
|
'onboarding-help',
|
|
],
|
|
cannotHandle: [
|
|
'moderation-report',
|
|
'sanctions',
|
|
'appeal',
|
|
'policy-enforcement',
|
|
],
|
|
domain: 'onboarding',
|
|
handoffTargets: ['moderation', 'supervisor'],
|
|
outputContract: {
|
|
format: 'text',
|
|
maxChars: '500',
|
|
requiredFields: 'text',
|
|
},
|
|
riskTier: 'medium',
|
|
};
|
|
|
|
export class OnboardingSkill implements Skill {
|
|
readonly name = 'onboarding';
|
|
readonly priority = 70;
|
|
private readonly classifier: OnboardingIntentClassifier;
|
|
private readonly blackboardStore: OnboardingBlackboardStore;
|
|
private readonly orchestrator: OnboardingOrchestrator;
|
|
private readonly shadowEvalPublisher: ShadowEvalPublisher;
|
|
private readonly toolRegistry: ToolRegistry;
|
|
private readonly tools: readonly Tool<unknown, unknown>[];
|
|
private readonly userMemoryStore: UserMemoryStore;
|
|
|
|
constructor(
|
|
private readonly garnetClient: GarnetClient,
|
|
private readonly llmClient: LlmClient,
|
|
private readonly roomContextService?: RoomContextService,
|
|
shadowEvalPublisher?: ShadowEvalPublisher,
|
|
private readonly natsClient?: Pick<NatsClient, 'request'>,
|
|
private readonly memoryService?: MemoryService,
|
|
private readonly escalationHistoryService?: EscalationHistoryService,
|
|
) {
|
|
this.classifier = new OnboardingIntentClassifier(llmClient);
|
|
this.blackboardStore = new OnboardingBlackboardStore(
|
|
garnetClient,
|
|
memoryService?.session,
|
|
);
|
|
this.userMemoryStore = new UserMemoryStore(
|
|
garnetClient,
|
|
memoryService?.longTerm,
|
|
);
|
|
this.orchestrator = new OnboardingOrchestrator(llmClient);
|
|
this.shadowEvalPublisher = shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
|
|
this.toolRegistry = createOnboardingToolRegistry();
|
|
this.tools = this.toolRegistry.list();
|
|
}
|
|
|
|
canHandle(request: AgentRequest): boolean {
|
|
return request.skill === 'onboarding';
|
|
}
|
|
|
|
capabilityCard(): SkillCapabilityCard {
|
|
return CAPABILITY_CARD;
|
|
}
|
|
|
|
async handle(
|
|
request: AgentRequest,
|
|
conversationHistory: ConversationMessage[],
|
|
): Promise<AgentResponse> {
|
|
if (getActiveProcessTrace()) {
|
|
return this.handleWithTrace(request, conversationHistory);
|
|
}
|
|
|
|
const collector = new ProcessTraceCollector();
|
|
return runWithProcessTrace(collector, () =>
|
|
this.handleWithTrace(request, conversationHistory),
|
|
);
|
|
}
|
|
|
|
private async handleWithTrace(
|
|
request: AgentRequest,
|
|
conversationHistory: ConversationMessage[],
|
|
): Promise<AgentResponse> {
|
|
const logger = getRequestLogger();
|
|
const trace = getActiveProcessTrace();
|
|
const session = await this.loadSession(request.chatroomId, request.userId);
|
|
|
|
if (request.trigger === 'user-message' && !request.userMessage?.trim()) {
|
|
return this.makeEmptyResponse(request.requestId);
|
|
}
|
|
|
|
const classifyStep = trace?.startStep('classify');
|
|
const classified = await this.classifier.classify(request.userMessage, request.trigger);
|
|
let classifyFinished = false;
|
|
const finishClassify = (abstention?: AbstentionDecision): void => {
|
|
if (classifyFinished) {
|
|
return;
|
|
}
|
|
classifyFinished = true;
|
|
classifyStep?.finish({
|
|
...(abstention ? { abstention } : {}),
|
|
confidence: classified.confidence,
|
|
intent: classified.intent.kind,
|
|
source: classified.source,
|
|
});
|
|
};
|
|
|
|
const deriveStep = trace?.startStep('derive');
|
|
const derived = deriveState(session, request.trigger, classified.intent);
|
|
const escalationHistory = await this.loadEscalationHistory(request.userId);
|
|
const routerMode = getAgentRouterMode();
|
|
if (routerMode === 'static' || routerMode === 'learned-v1') {
|
|
const routerInput = {
|
|
classified,
|
|
derived,
|
|
intentRiskTier: this.capabilityCard().riskTier,
|
|
recentLatencyP95Ms: this.getRecentLatencyP95Ms(request),
|
|
request,
|
|
session,
|
|
userEscalationCount: escalationHistory?.totalEscalations ?? 0,
|
|
};
|
|
derived.routingDecision =
|
|
routerMode === 'learned-v1'
|
|
? decideHybridRouting(routerInput, {
|
|
learnedModel: loadLearnedRouterModel(getAgentRouterWeightsPath()),
|
|
})
|
|
: decidePreGenerationRouting(routerInput);
|
|
}
|
|
deriveStep?.finish({
|
|
phase: derived.phase.kind,
|
|
responseMode: derived.responseMode,
|
|
routingDecision: derived.routingDecision,
|
|
shouldRespond: derived.shouldRespond,
|
|
tier: derived.tier,
|
|
});
|
|
|
|
if (getAgentAbstentionEnabled()) {
|
|
const decision = await this.decideEarlyAbstention(request, classified, derived);
|
|
derived.abstentionScore = decision.score;
|
|
finishClassify(decision);
|
|
|
|
if (decision.action.kind === 'no-response') {
|
|
const response = this.makeEmptyResponse(request.requestId);
|
|
trace?.startStep('respond').finish({
|
|
draftText: response.text,
|
|
tokensIn: 0,
|
|
tokensOut: 0,
|
|
});
|
|
validateOnboardingResponse('[NO_RESPONSE]', derived, {
|
|
allowShortResponse: true,
|
|
});
|
|
this.shadowEvalPublisher.publish({
|
|
classified,
|
|
createdAt: new Date().toISOString(),
|
|
derived,
|
|
latencyMs: this.getCurrentLatencyMs(),
|
|
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
|
request,
|
|
response,
|
|
skill: this.name,
|
|
});
|
|
return response;
|
|
}
|
|
|
|
if (decision.action.kind === 'clarify') {
|
|
const response = this.makeTemplateResponse(
|
|
request.requestId,
|
|
getOnboardingTemplate(decision.action.templateKey),
|
|
);
|
|
trace?.startStep('respond').finish({
|
|
draftText: response.text,
|
|
tokensIn: 0,
|
|
tokensOut: 0,
|
|
});
|
|
validateOnboardingResponse(response.text, derived, {
|
|
allowShortResponse: true,
|
|
});
|
|
|
|
await this.rememberTurn(request, derived).catch((error) => {
|
|
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
|
|
});
|
|
await this.observeUserMemory(request, derived, response).catch((error) => {
|
|
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
|
|
});
|
|
|
|
this.shadowEvalPublisher.publish({
|
|
classified,
|
|
createdAt: new Date().toISOString(),
|
|
derived,
|
|
latencyMs: this.getCurrentLatencyMs(),
|
|
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
|
request,
|
|
response,
|
|
skill: this.name,
|
|
});
|
|
|
|
return response;
|
|
}
|
|
} else {
|
|
finishClassify();
|
|
}
|
|
|
|
if (!derived.shouldRespond) {
|
|
const response = this.makeEmptyResponse(request.requestId);
|
|
trace?.startStep('respond').finish({
|
|
draftText: response.text,
|
|
tokensIn: 0,
|
|
tokensOut: 0,
|
|
});
|
|
validateOnboardingResponse('[NO_RESPONSE]', derived, {
|
|
allowShortResponse: true,
|
|
});
|
|
return response;
|
|
}
|
|
|
|
let response: AgentResponse;
|
|
|
|
if (derived.responseMode === 'template') {
|
|
response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived);
|
|
} else {
|
|
const llmResult = await this.handleLlm(request, session, conversationHistory, derived);
|
|
|
|
if (llmResult.status === 'success') {
|
|
response = this.makeLlmResponse(request.requestId, llmResult.text);
|
|
} else if (llmResult.status === 'no_response') {
|
|
response = this.makeEmptyResponse(request.requestId);
|
|
} else {
|
|
response = this.makeTemplateFallbackResponse(request, session, derived.intent, derived);
|
|
}
|
|
}
|
|
|
|
trace?.startStep('respond').finish({
|
|
draftText: response.text,
|
|
tokensIn: 0,
|
|
tokensOut: 0,
|
|
});
|
|
validateOnboardingResponse(response.text.trim() ? response.text : '[NO_RESPONSE]', derived, {
|
|
allowShortResponse: derived.allowShortResponse,
|
|
});
|
|
|
|
if (derived.intent.kind === 'discord-verify-question') {
|
|
logger.debug(
|
|
{
|
|
responseSource: response.source,
|
|
responseText: response.text,
|
|
},
|
|
'[OnboardingSkill] Final discord verification response',
|
|
);
|
|
}
|
|
|
|
if (response.text.trim()) {
|
|
await this.rememberTurn(request, derived).catch((error) => {
|
|
logger.warn({ err: error }, '[OnboardingSkill] Failed to update blackboard');
|
|
});
|
|
await this.observeUserMemory(request, derived, response).catch((error) => {
|
|
logger.warn({ err: error }, '[OnboardingSkill] Failed to update user memory');
|
|
});
|
|
|
|
this.shadowEvalPublisher.publish({
|
|
classified,
|
|
createdAt: new Date().toISOString(),
|
|
derived,
|
|
latencyMs: this.getCurrentLatencyMs(),
|
|
process: trace ? anonymizeProcessTrace(trace.snapshot()) : undefined,
|
|
request,
|
|
response,
|
|
skill: this.name,
|
|
});
|
|
}
|
|
|
|
return response;
|
|
}
|
|
|
|
private async loadSession(chatroomId: string, userId: string): Promise<OnboardingSession | null> {
|
|
const prefix = getOnboardingSessionKeyPrefix();
|
|
const key = `${prefix}:onboarding:${chatroomId}:${userId}`;
|
|
const rawSession = await this.garnetClient.getJson<StoredOnboardingSession>(key);
|
|
return normalizeOnboardingSession(rawSession);
|
|
}
|
|
|
|
private async loadEscalationHistory(userId: string) {
|
|
if (!this.escalationHistoryService) {
|
|
return null;
|
|
}
|
|
try {
|
|
return await this.escalationHistoryService.load(userId);
|
|
} catch (error) {
|
|
getRequestLogger().warn(
|
|
{ err: error },
|
|
'[OnboardingSkill] Failed to load escalation history',
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private getRecentLatencyP95Ms(request: AgentRequest): number | undefined {
|
|
const raw = request.context?.['recentLatencyP95Ms'];
|
|
return typeof raw === 'number' && Number.isFinite(raw) ? raw : undefined;
|
|
}
|
|
|
|
private async decideEarlyAbstention(
|
|
request: AgentRequest,
|
|
classified: ClassifiedIntent,
|
|
derived: DerivedState,
|
|
): Promise<AbstentionDecision> {
|
|
const logger = getRequestLogger();
|
|
let blackboard: Blackboard | null = null;
|
|
|
|
try {
|
|
blackboard = await this.blackboardStore.load(request.chatroomId, request.userId);
|
|
} catch (error) {
|
|
logger.warn({ err: error }, '[OnboardingSkill] Failed to load blackboard for abstention');
|
|
}
|
|
|
|
return decideEarlyAbstention({
|
|
blackboard,
|
|
classified,
|
|
derived,
|
|
request,
|
|
thresholds: {
|
|
abstain: getAgentAbstentionThreshold(),
|
|
clarify: getAgentAbstentionClarifyThreshold(),
|
|
},
|
|
});
|
|
}
|
|
|
|
private async handleLlm(
|
|
request: AgentRequest,
|
|
session: OnboardingSession | null,
|
|
conversationHistory: ConversationMessage[],
|
|
derived: DerivedState,
|
|
): Promise<LlmTextResult> {
|
|
if (!this.llmClient.hasEndpoints()) {
|
|
return { status: 'failed' };
|
|
}
|
|
|
|
const messages = await buildLlmMessages(
|
|
{
|
|
blackboardStore: this.blackboardStore,
|
|
garnetClient: this.garnetClient,
|
|
memoryService: this.memoryService,
|
|
roomContextService: this.roomContextService,
|
|
userMemoryStore: this.userMemoryStore,
|
|
},
|
|
{
|
|
conversationHistory,
|
|
derived,
|
|
request,
|
|
session,
|
|
},
|
|
);
|
|
|
|
return generateLlmText(this.orchestrator, messages, derived, {
|
|
allowShortResponse: derived.allowShortResponse,
|
|
maxTokens: derived.suggestedMaxTokens,
|
|
request,
|
|
retryOnLowQuality:
|
|
request.trigger === 'user-message' &&
|
|
!derived.allowShortResponse &&
|
|
derived.responseMode !== 'llm-with-tools',
|
|
toolContext: this.buildToolContext(request, session),
|
|
tools: selectToolsForDerived(derived, this.tools, this.toolRegistry),
|
|
});
|
|
}
|
|
|
|
private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise<void> {
|
|
return rememberTurn(this.blackboardStore, request, derived);
|
|
}
|
|
|
|
private async observeUserMemory(
|
|
request: AgentRequest,
|
|
derived: DerivedState,
|
|
response: AgentResponse,
|
|
): Promise<void> {
|
|
if (!getAgentUserMemoryEnabled()) {
|
|
return;
|
|
}
|
|
|
|
const profile = await this.userMemoryStore.observeTurn(request.userId, derived, response, {
|
|
flaggedIssues:
|
|
derived.phase.kind === 'nickname-rejected'
|
|
? derived.phase.issues.map((issue) => issue.message)
|
|
: [],
|
|
});
|
|
if (!shouldRefreshLongTermSummary(profile)) {
|
|
return;
|
|
}
|
|
|
|
void this.refreshLongTermSummary(request, profile);
|
|
}
|
|
|
|
private async refreshLongTermSummary(
|
|
request: AgentRequest,
|
|
profile: Awaited<ReturnType<UserMemoryStore['observeTurn']>>,
|
|
): Promise<void> {
|
|
try {
|
|
const conversationSummary =
|
|
(await this.memoryService?.session.getSummary(
|
|
request.chatroomId,
|
|
request.userId,
|
|
)) ?? null;
|
|
const longTermSummary = await buildLongTermSummary(
|
|
this.llmClient,
|
|
profile,
|
|
conversationSummary,
|
|
);
|
|
await this.userMemoryStore.updateProfile(request.userId, {
|
|
longTermSummary,
|
|
});
|
|
} catch (error) {
|
|
getRequestLogger().warn(
|
|
{ err: error },
|
|
'[OnboardingSkill] Failed to refresh long-term user summary',
|
|
);
|
|
}
|
|
}
|
|
|
|
private makeTemplateFallbackResponse(
|
|
request: AgentRequest,
|
|
session: OnboardingSession | null,
|
|
intent: Intent,
|
|
derived: DerivedState,
|
|
): AgentResponse {
|
|
switch (request.trigger) {
|
|
case 'new-member':
|
|
return this.makeEmptyResponse(request.requestId);
|
|
case 'nickname-invalid':
|
|
return this.makeTemplateResponse(
|
|
request.requestId,
|
|
getNicknameInvalidMessage(request.nickname, session),
|
|
);
|
|
case 'photo-failed':
|
|
return this.makeTemplateResponse(
|
|
request.requestId,
|
|
getPhotoFailedMessage(request.nickname),
|
|
);
|
|
case 'progress-update':
|
|
return this.makeTemplateResponse(
|
|
request.requestId,
|
|
session
|
|
? getProgressMessage(request.nickname, session)
|
|
: getGenericHelpMessage(request.nickname, session),
|
|
);
|
|
case 'timeout-warning':
|
|
return this.makeTemplateResponse(
|
|
request.requestId,
|
|
getTimeoutWarningMessage(request.nickname),
|
|
);
|
|
case 'completed':
|
|
return this.makeTemplateResponse(request.requestId, getCompletedMessage(request.nickname));
|
|
default:
|
|
return this.makeTemplateResponse(
|
|
request.requestId,
|
|
getIntentFallbackText(request.nickname, session, intent, derived),
|
|
);
|
|
}
|
|
}
|
|
|
|
private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext {
|
|
return {
|
|
chatroomId: request.chatroomId,
|
|
natsClient: this.natsClient,
|
|
userId: request.userId,
|
|
session,
|
|
garnetClient: this.garnetClient,
|
|
memoryService: this.memoryService,
|
|
roomContextService: this.roomContextService,
|
|
toolRegistry: this.toolRegistry,
|
|
};
|
|
}
|
|
|
|
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
|
|
return {
|
|
requestId,
|
|
text,
|
|
confidence: 0.8,
|
|
source: 'template',
|
|
};
|
|
}
|
|
|
|
private makeLlmResponse(requestId: string, text: string): AgentResponse {
|
|
return {
|
|
requestId,
|
|
text,
|
|
confidence: 0.9,
|
|
source: 'llm',
|
|
};
|
|
}
|
|
|
|
private makeEmptyResponse(requestId: string): AgentResponse {
|
|
return {
|
|
requestId,
|
|
text: '',
|
|
confidence: 0,
|
|
source: 'template',
|
|
};
|
|
}
|
|
|
|
private getCurrentLatencyMs(): number | undefined {
|
|
const startedAt = getContext()?.startedAt;
|
|
const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN;
|
|
if (!Number.isFinite(startedTimestamp)) {
|
|
return undefined;
|
|
}
|
|
|
|
return Math.max(0, Date.now() - startedTimestamp);
|
|
}
|
|
}
|