Iterate skill SSOT routing and eval support

Add skill priority routing and duplicate manifest handling.

Split eval golden sets by skill and add moderation rule lookup coverage.

Move onboarding runtime helpers out of the main skill class.
This commit is contained in:
dahyeon committed 2026-05-18 09:19:17 -07:00
1 parent fb189330da
commit 12e731a428
35 files changed
+1260 -500

No files matched your search

+31 -354
View File
@@ -1,38 +1,22 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from '@agent/types';
import { getConversationSummaryKey } from '@conv/conversation-store';
import type { GarnetClient } from '@common/garnet';
import type { NatsClient } from '@common/nats';
import { getContext, getRequestLogger } from '@obs/context';
import type { LlmClient } from '@llm/llm-client';
import type { ChatMessage } from '@llm/types';
import type { RoomContextService } from '@room/room-context-service';
import type { Skill } from '@skills/skill';
import { type ShadowEvalPublisher, ShadowEvalBus } from '@eval/shadow-bus';
import { anonymizeProcessTrace } from '@eval/anonymize';
import { meetsTrustTier } from '../../eval/governance/trust.js';
import { createOnboardingToolRegistry } from './tools.js';
import type { ToolRegistry } from '@tools/registry';
import type { Tool, ToolContext } from '@tools/tool';
import { createToolSearchTool } from '@tools/tool-search';
import {
loadPromptVariantManifest,
PromptBuilder,
} from './prompts/builder.js';
import {
getAcknowledgementMessage,
getAgeRequirementMessage,
getCompletedMessage,
getDiscordMessage,
getGreetingMessage,
getGenericHelpMessage,
getHeartGuideMessage,
getHeartLocationMessage,
getNicknameInvalidMessage,
getNicknameFormatMessage,
getOnboardingTemplate,
getPhotoFailedMessage,
getProgressMessage,
getRemainingStatusMessage,
getTimeoutWarningMessage,
} from './onboarding-templates.js';
import {
@@ -41,13 +25,8 @@ import {
getAgentAbstentionThreshold,
getAgentRouterMode,
getAgentRouterWeightsPath,
getAgentTrustMinTier,
getAgentToolSearchEnabled,
getAgentToolSearchThreshold,
getAgentUserMemoryEnabled,
getOnboardingSessionKeyPrefix,
getLlmTemperature,
getPromptVariantId,
} from '../../config.js';
import { OnboardingBlackboardStore } from './state/blackboard.js';
import { UserMemoryStore } from './state/user-memory.js';
@@ -74,21 +53,24 @@ import {
runWithProcessTrace,
} from './runtime/trace.js';
import { validateOnboardingResponse } from './runtime/validator.js';
import { getIntentFallbackText } from './runtime/fallback-text-resolver.js';
import { buildLlmMessages } from './runtime/message-builder.js';
import {
generateLlmText,
type LlmTextResult,
} from './runtime/text-generator.js';
import { selectToolsForDerived } from './runtime/tool-selector.js';
import { rememberTurn } from './runtime/state-updater.js';
import { normalizeOnboardingSession } from './session.js';
import type {
Blackboard,
OnboardingSession,
PromptContext,
StoredOnboardingSession,
} from './types.js';
type LlmTextResult =
| { status: 'success'; text: string }
| { status: 'no_response' }
| { status: 'failed' };
export class OnboardingSkill implements Skill {
readonly name = 'onboarding';
readonly priority = 70;
private readonly classifier: OnboardingIntentClassifier;
private readonly blackboardStore: OnboardingBlackboardStore;
private readonly orchestrator: OnboardingOrchestrator;
@@ -357,9 +339,26 @@ export class OnboardingSkill implements Skill {
conversationHistory: ConversationMessage[],
derived: DerivedState,
): Promise<LlmTextResult> {
const messages = await this.buildLlmMessages(request, session, conversationHistory, derived);
if (!this.llmClient.hasEndpoints()) {
return { status: 'failed' };
}
return this.generateLlmText(messages, derived, {
const messages = await buildLlmMessages(
{
blackboardStore: this.blackboardStore,
garnetClient: this.garnetClient,
roomContextService: this.roomContextService,
userMemoryStore: this.userMemoryStore,
},
{
conversationHistory,
derived,
request,
session,
},
);
return generateLlmText(this.orchestrator, messages, derived, {
allowShortResponse: derived.allowShortResponse,
maxTokens: derived.suggestedMaxTokens,
request,
@@ -368,135 +367,12 @@ export class OnboardingSkill implements Skill {
!derived.allowShortResponse &&
derived.responseMode !== 'llm-with-tools',
toolContext: this.buildToolContext(request, session),
tools: this.selectToolsForDerived(derived),
tools: selectToolsForDerived(derived, this.tools, this.toolRegistry),
});
}
private async buildLlmMessages(
request: AgentRequest,
session: OnboardingSession | null,
conversationHistory: ConversationMessage[],
derived: DerivedState,
): Promise<ChatMessage[]> {
const logger = getRequestLogger();
const roomContextPromise =
request.trigger === 'user-message' && this.roomContextService
? this.roomContextService.getRecentMessages(request.chatroomId).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to fetch room context');
return undefined;
})
: Promise.resolve(undefined);
const userProfilePromise = getAgentUserMemoryEnabled()
? this.userMemoryStore.load(request.userId).catch((error) => {
logger.warn({ err: error }, '[OnboardingSkill] Failed to load user memory');
return null;
})
: Promise.resolve(null);
const [blackboard, summary, roomContext, userProfile] = await Promise.all([
this.blackboardStore.load(request.chatroomId, request.userId),
this.loadConversationSummary(request.chatroomId, request.userId),
roomContextPromise,
userProfilePromise,
]);
const context: PromptContext = {
blackboard,
conversationHistory,
roomContext,
summary,
userMessage: this.buildPromptUserMessage(request, derived),
userProfile,
};
const builder = PromptBuilder.forDerived(derived, session, context);
const promptVariantId = getPromptVariantId();
if (!promptVariantId) {
return builder.build();
}
const manifest = loadPromptVariantManifest(promptVariantId);
if (!manifest || !meetsTrustTier(manifest.trustTier, getAgentTrustMinTier())) {
return builder.build();
}
return builder.applyVariant(manifest).build();
}
private buildPromptUserMessage(request: AgentRequest, derived: DerivedState): string {
switch (request.trigger) {
case 'nickname-invalid':
return `현재 닉네임은 "${request.nickname}"예요. 왜 안 되는지와 어디를 고치면 되는지만 알려줘.`;
case 'photo-failed':
return '하트 인증 사진이 실패했어요. 다시 시도할 수 있게 필요한 내용만 안내해줘.';
case 'progress-update':
return '현재 새내기 등록 진행상황만 짧게 알려줘.';
case 'timeout-warning':
return '등록 마감이 가까워졌어요. 남은 절차만 짧고 따뜻하게 안내해줘.';
case 'completed':
return '새내기 등록을 모두 마친 유저에게 짧고 따뜻하게 축하 인사를 해줘.';
case 'user-message':
if (derived.intent.kind === 'status-question') {
return '완료된 것은 빼고 남은 절차만 짧게 말해줘.';
}
return request.userMessage?.trim() || '지금 필요한 온보딩 안내만 도와줘.';
default:
return '지금 필요한 온보딩 안내만 짧게 도와줘.';
}
}
private async loadConversationSummary(
chatroomId: string,
userId: string,
): Promise<string | undefined> {
const key = getConversationSummaryKey(chatroomId, userId);
return (await this.garnetClient.getString(key)) ?? undefined;
}
private async rememberTurn(request: AgentRequest, derived: DerivedState): Promise<void> {
const now = new Date().toISOString();
const userMessage = request.userMessage?.trim();
await this.blackboardStore.update(request.chatroomId, request.userId, (current) => {
const sameIntentStreak =
current.lastIntent === derived.intent.kind
? (current.sameIntentStreak ?? 1) + 1
: 1;
const openQuestions =
userMessage &&
(derived.intent.kind === 'help-how' ||
derived.intent.kind === 'help-where' ||
derived.intent.kind === 'nickname-question' ||
derived.intent.kind === 'photo-question' ||
derived.intent.kind === 'discord-question' ||
derived.intent.kind === 'discord-verify-question')
? [...current.openQuestions, userMessage].slice(-3)
: current.openQuestions;
const recentNicknameAttempts =
request.trigger === 'nickname-invalid' || derived.intent.kind === 'nickname-question'
? Array.from(new Set([...current.recentNicknameAttempts, request.nickname])).slice(-5)
: current.recentNicknameAttempts;
return {
...current,
flaggedIssues:
derived.phase.kind === 'nickname-rejected'
? derived.phase.issues.map((issue) => issue.message)
: derived.phase.kind === 'completed'
? []
: current.flaggedIssues,
lastIntent: derived.intent.kind,
lastIntentAt: now,
openQuestions: derived.phase.kind === 'completed' ? [] : openQuestions,
recentNicknameAttempts,
sameIntentStreak: derived.phase.kind === 'completed' ? 0 : sameIntentStreak,
toneMemory: {
...current.toneMemory,
greetedAt: derived.intent.kind === 'greeting' ? now : current.toneMemory.greetedAt,
thankedAt: derived.intent.kind === 'acknowledgement' ? now : current.toneMemory.thankedAt,
},
};
});
return rememberTurn(this.blackboardStore, request, derived);
}
private async observeUserMemory(
@@ -552,163 +428,11 @@ export class OnboardingSkill implements Skill {
default:
return this.makeTemplateResponse(
request.requestId,
this.getIntentFallbackText(request.nickname, session, intent, derived),
getIntentFallbackText(request.nickname, session, intent, derived),
);
}
}
private getIntentFallbackText(
nickname: string,
session: OnboardingSession | null,
intent: Intent,
derived: DerivedState,
): string {
switch (intent.kind) {
case 'greeting':
return getGreetingMessage();
case 'acknowledgement':
return getAcknowledgementMessage();
case 'status-question':
return getRemainingStatusMessage(session);
case 'nickname-question':
return derived.phase.kind === 'nickname-rejected'
? getNicknameInvalidMessage(nickname, session)
: getNicknameFormatMessage();
case 'photo-question':
return derived.phase.kind === 'photo-rejected'
? getPhotoFailedMessage(nickname)
: getHeartGuideMessage();
case 'help-where':
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
) {
return getHeartLocationMessage();
}
return getGenericHelpMessage(nickname, session);
case 'help-how':
if (
derived.phase.kind === 'nickname-pending' ||
derived.phase.kind === 'nickname-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('nickname'))
) {
return getNicknameFormatMessage();
}
if (
derived.phase.kind === 'photo-pending' ||
derived.phase.kind === 'photo-rejected' ||
(derived.phase.kind === 'deadline-near' && derived.phase.remaining.includes('photo'))
) {
return getHeartGuideMessage();
}
return getGenericHelpMessage(nickname, session);
case 'discord-question':
return getDiscordMessage(nickname, session);
case 'discord-verify-question':
return getDiscordMessage(nickname, session);
case 'age-question':
return getAgeRequirementMessage();
case 'chitchat':
case 'off-topic-to-other':
return '';
case 'unknown':
return getGenericHelpMessage(nickname, session);
}
}
private async generateLlmText(
messages: ChatMessage[],
derived: DerivedState,
options?: {
allowShortResponse?: boolean;
maxTokens?: number;
request: AgentRequest;
retryOnLowQuality?: boolean;
temperature?: number;
toolContext?: ToolContext | undefined;
tools?: readonly Tool<unknown, unknown>[] | undefined;
},
): Promise<LlmTextResult> {
if (!this.llmClient.hasEndpoints()) {
return { status: 'failed' };
}
const attempts: Array<{
maxTokens?: number;
temperature?: number;
}> = [];
const firstAttempt: {
maxTokens?: number;
temperature?: number;
} = {};
if (options?.maxTokens !== undefined) {
firstAttempt.maxTokens = options.maxTokens;
}
if (options?.temperature !== undefined) {
firstAttempt.temperature = options.temperature;
}
attempts.push(firstAttempt);
if (options?.retryOnLowQuality) {
const retryAttempt: {
maxTokens?: number;
temperature?: number;
} = {
temperature: getLlmTemperature() + 0.1,
};
if (options.maxTokens !== undefined) {
retryAttempt.maxTokens = options.maxTokens;
}
attempts.push(retryAttempt);
}
for (const attempt of attempts) {
const requestOptions: {
maxTokens?: number;
temperature?: number;
} = {};
if (attempt.maxTokens !== undefined) {
requestOptions.maxTokens = attempt.maxTokens;
}
if (attempt.temperature !== undefined) {
requestOptions.temperature = attempt.temperature;
}
const toolContext = options?.toolContext ?? {};
const llmText = await this.orchestrator.run(derived, messages, {
maxTokens: requestOptions.maxTokens,
request: options?.request,
temperature: requestOptions.temperature,
toolContext,
tools: options?.tools,
});
if (!llmText) {
continue;
}
const validation = validateOnboardingResponse(llmText, derived, {
allowShortResponse: options?.allowShortResponse,
});
if (validation.text.includes('[NO_RESPONSE]')) {
return { status: 'no_response' };
}
if (validation.status === 'reject') {
continue;
}
return {
status: 'success',
text: validation.text,
};
}
return { status: 'failed' };
}
private buildToolContext(request: AgentRequest, session: OnboardingSession | null): ToolContext {
return {
chatroomId: request.chatroomId,
@@ -721,53 +445,6 @@ export class OnboardingSkill implements Skill {
};
}
private selectToolsForDerived(derived: DerivedState): readonly Tool<unknown, unknown>[] {
const maybeUseToolSearch = (
tools: readonly Tool<unknown, unknown>[],
): readonly Tool<unknown, unknown>[] => {
if (
getAgentToolSearchEnabled() &&
tools.length >= getAgentToolSearchThreshold()
) {
return [createToolSearchTool(this.toolRegistry)];
}
return tools;
};
if (derived.intent.kind === 'discord-verify-question') {
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'verify-discord-nickname'),
);
}
if (derived.phase.kind === 'nickname-rejected') {
return maybeUseToolSearch(
this.tools.filter(
(tool) =>
tool.name === 'validate-nickname' ||
tool.name === 'lookup-community-rule' ||
tool.name === 'check-onboarding-progress',
),
);
}
if (derived.intent.kind === 'status-question') {
return maybeUseToolSearch(
this.tools.filter((tool) => tool.name === 'check-onboarding-progress'),
);
}
if (derived.intent.kind === 'discord-question') {
return maybeUseToolSearch(
this.tools.filter(
(tool) => tool.name === 'lookup-community-rule' || tool.name === 'verify-discord-nickname',
),
);
}
return maybeUseToolSearch(this.tools);
}
private makeTemplateResponse(requestId: string, text: string): AgentResponse {
return {
requestId,