Add eval runner and tool orchestration

This commit is contained in:
dahyeon committed 2026-04-17 00:47:02 -07:00
1 parent 4294f3c92e
commit 5579db54da
66 files changed
+6162 -242

No files matched your search

+100 -91
View File
@@ -1,11 +1,17 @@
import type { AgentRequest, AgentResponse, ConversationMessage } from "@agent/types";
import { getConversationSummaryKey } from "@conv/conversation-store";
import type { GarnetClient } from "@common/garnet";
import { getRequestLogger } from "@obs/context";
import { getContext, getRequestLogger } from "@obs/context";
import type { LlmClient } from "@llm/llm-client";
import type { ChatMessage } from "@llm/types";
import type { RoomContextService } from "@room/room-context-service";
import type { Skill } from "@skills/skill";
import {
type ShadowEvalPublisher,
ShadowEvalBus,
} from "@eval/shadow-bus";
import { createBuiltInToolRegistry } from "@tools/builtins/index";
import type { Tool, ToolContext } from "@tools/tool";
import { PromptBuilder } from "./prompts/builder.js";
import {
getAcknowledgementMessage,
@@ -23,103 +29,40 @@ import {
getRemainingStatusMessage,
getTimeoutWarningMessage,
} from "./onboarding-templates.js";
import { getOnboardingSessionKeyPrefix, getMaxResponseLength, getLlmTemperature } from "../../config.js";
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from "../../config.js";
import { OnboardingBlackboardStore } from "./state/blackboard.js";
import { deriveState } from "./state/derive.js";
import { OnboardingIntentClassifier } from "./state/classifier.js";
import type { DerivedState, Intent } from "./state/onboarding-state.js";
import { OnboardingOrchestrator } from "./runtime/orchestrator.js";
import { validateOnboardingResponse } from "./runtime/validator.js";
import type { OnboardingSession, PromptContext } from "./types.js";
const ALLOWED_URL_PATTERNS = [
/discord\.gg\/lolympus/u,
/open\.kakao\.com\/o\/gOscBAEh/u,
];
type LlmTextResult =
| { status: "success"; text: string }
| { status: "no_response" }
| { status: "failed" };
function postProcessResponse(text: string): string {
let processed = text.trim();
// Remove <think> blocks (Qwen thinking mode artifacts)
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
// Strip URLs not in allowlist
processed = processed.replace(
/https?:\/\/[^\s)}\]]+/gu,
(match) => {
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
return match;
}
return "";
},
);
// Clean up double spaces from URL removal
processed = processed.replace(/ +/gu, " ").trim();
// Strip markdown formatting (KakaoTalk is plaintext)
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1"); // **bold**
processed = processed.replace(/\*(.+?)\*/gu, "$1"); // *italic*
processed = processed.replace(/__(.+?)__/gu, "$1"); // __underline__
processed = processed.replace(/~~(.+?)~~/gu, "$1"); // ~~strike~~
processed = processed.replace(/^#{1,6}\s+/gmu, ""); // ## headings
processed = processed.replace(/^[-*]\s+/gmu, ""); // - or * list items
processed = processed.replace(/`(.+?)`/gu, "$1"); // `inline code`
// Truncate to max length with Korean sentence boundary awareness
const maxLen = getMaxResponseLength();
if (processed.length > maxLen) {
processed = processed.slice(0, maxLen).trimEnd();
// Search backwards for a Korean sentence-ending pattern
const sentenceEndPattern = /(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
// Try progressively shorter substrings to find a sentence boundary
for (let i = processed.length; i > maxLen * 0.5; i--) {
const sub = processed.slice(0, i);
if (sentenceEndPattern.test(sub)) {
processed = sub;
break;
}
}
}
return processed.trim();
}
function isLowQualityResponse(
text: string,
options?: { allowShortResponse?: boolean },
): boolean {
if (!options?.allowShortResponse && text.length < 10) return true;
// Check for excessive repetition (same phrase 3+ times)
const words = text.split(/\s+/u);
if (words.length >= 6) {
const phrases = new Map<string, number>();
for (let i = 0; i < words.length - 1; i++) {
const bigram = `${words[i]} ${words[i + 1]}`;
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
}
for (const count of phrases.values()) {
if (count >= 3) return true;
}
}
return false;
}
export class OnboardingSkill implements Skill {
readonly name = "onboarding";
private readonly classifier: OnboardingIntentClassifier;
private readonly blackboardStore: OnboardingBlackboardStore;
private readonly orchestrator: OnboardingOrchestrator;
private readonly shadowEvalPublisher: ShadowEvalPublisher;
private readonly tools: readonly Tool<unknown, unknown>[];
constructor(
private readonly garnetClient: GarnetClient,
private readonly llmClient: LlmClient,
private readonly roomContextService?: RoomContextService,
shadowEvalPublisher?: ShadowEvalPublisher,
) {
this.classifier = new OnboardingIntentClassifier(llmClient);
this.blackboardStore = new OnboardingBlackboardStore(garnetClient);
this.orchestrator = new OnboardingOrchestrator(llmClient);
this.shadowEvalPublisher =
shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
this.tools = createBuiltInToolRegistry().list();
}
canHandle(request: AgentRequest): boolean {
@@ -185,6 +128,15 @@ export class OnboardingSkill implements Skill {
await this.rememberTurn(request, derived).catch((error) => {
logger.warn({ err: error }, "[OnboardingSkill] Failed to update blackboard");
});
this.shadowEvalPublisher.publish({
classified,
createdAt: new Date().toISOString(),
derived,
latencyMs: this.getCurrentLatencyMs(),
request,
response,
});
}
return response;
@@ -212,11 +164,15 @@ export class OnboardingSkill implements Skill {
derived,
);
return this.generateLlmText(messages, {
return this.generateLlmText(messages, derived, {
allowShortResponse: derived.allowShortResponse,
maxTokens: derived.suggestedMaxTokens,
retryOnLowQuality:
request.trigger === "user-message" && !derived.allowShortResponse,
request.trigger === "user-message" &&
!derived.allowShortResponse &&
derived.responseMode !== "llm-with-tools",
toolContext: this.buildToolContext(request, session),
tools: this.selectToolsForDerived(derived),
});
}
@@ -456,11 +412,14 @@ export class OnboardingSkill implements Skill {
private async generateLlmText(
messages: ChatMessage[],
derived: DerivedState,
options?: {
allowShortResponse?: boolean;
maxTokens?: number;
retryOnLowQuality?: boolean;
temperature?: number;
toolContext?: ToolContext | undefined;
tools?: readonly Tool<unknown, unknown>[] | undefined;
},
): Promise<LlmTextResult> {
if (!this.llmClient.hasEndpoints()) {
@@ -509,39 +468,79 @@ export class OnboardingSkill implements Skill {
requestOptions.temperature = attempt.temperature;
}
const llmResult = await this.llmClient.chatCompletion(
const toolContext = options?.toolContext ?? {};
const llmText = await this.orchestrator.run(
derived,
messages,
requestOptions,
{
maxTokens: requestOptions.maxTokens,
temperature: requestOptions.temperature,
toolContext,
tools: options?.tools,
},
);
if (!llmResult) {
if (!llmText) {
continue;
}
const processed = postProcessResponse(llmResult.content);
if (!processed) {
continue;
}
if (processed.includes("[NO_RESPONSE]")) {
const validation = validateOnboardingResponse(
llmText,
derived,
{
allowShortResponse: options?.allowShortResponse,
},
);
if (validation.text.includes("[NO_RESPONSE]")) {
return { status: "no_response" };
}
const qualityOptions =
options?.allowShortResponse !== undefined
? { allowShortResponse: options.allowShortResponse }
: undefined;
if (isLowQualityResponse(processed, qualityOptions)) {
if (validation.status === "reject") {
continue;
}
return {
status: "success",
text: processed,
text: validation.text,
};
}
return { status: "failed" };
}
private buildToolContext(
request: AgentRequest,
session: OnboardingSession | null,
): ToolContext {
return {
chatroomId: request.chatroomId,
userId: request.userId,
session,
garnetClient: this.garnetClient,
roomContextService: this.roomContextService,
};
}
private selectToolsForDerived(
derived: DerivedState,
): readonly Tool<unknown, unknown>[] {
if (derived.phase.kind === "nickname-rejected") {
return this.tools.filter((tool) =>
tool.name === "validate-nickname" ||
tool.name === "lookup-community-rule" ||
tool.name === "check-onboarding-progress",
);
}
if (derived.intent.kind === "status-question") {
return this.tools.filter((tool) =>
tool.name === "check-onboarding-progress",
);
}
return this.tools;
}
private makeTemplateResponse(
requestId: string,
text: string,
@@ -574,4 +573,14 @@ export class OnboardingSkill implements Skill {
source: "template",
};
}
private getCurrentLatencyMs(): number | undefined {
const startedAt = getContext()?.startedAt;
const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN;
if (!Number.isFinite(startedTimestamp)) {
return undefined;
}
return Math.max(0, Date.now() - startedTimestamp);
}
}