Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
+100
-91
@@ -1,11 +1,17 @@
|
||||
import type { AgentRequest, AgentResponse, ConversationMessage } from "@agent/types";
|
||||
import { getConversationSummaryKey } from "@conv/conversation-store";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import { getContext, getRequestLogger } from "@obs/context";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { RoomContextService } from "@room/room-context-service";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import {
|
||||
type ShadowEvalPublisher,
|
||||
ShadowEvalBus,
|
||||
} from "@eval/shadow-bus";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import { PromptBuilder } from "./prompts/builder.js";
|
||||
import {
|
||||
getAcknowledgementMessage,
|
||||
@@ -23,103 +29,40 @@ import {
|
||||
getRemainingStatusMessage,
|
||||
getTimeoutWarningMessage,
|
||||
} from "./onboarding-templates.js";
|
||||
import { getOnboardingSessionKeyPrefix, getMaxResponseLength, getLlmTemperature } from "../../config.js";
|
||||
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from "../../config.js";
|
||||
import { OnboardingBlackboardStore } from "./state/blackboard.js";
|
||||
import { deriveState } from "./state/derive.js";
|
||||
import { OnboardingIntentClassifier } from "./state/classifier.js";
|
||||
import type { DerivedState, Intent } from "./state/onboarding-state.js";
|
||||
import { OnboardingOrchestrator } from "./runtime/orchestrator.js";
|
||||
import { validateOnboardingResponse } from "./runtime/validator.js";
|
||||
import type { OnboardingSession, PromptContext } from "./types.js";
|
||||
|
||||
const ALLOWED_URL_PATTERNS = [
|
||||
/discord\.gg\/lolympus/u,
|
||||
/open\.kakao\.com\/o\/gOscBAEh/u,
|
||||
];
|
||||
|
||||
type LlmTextResult =
|
||||
| { status: "success"; text: string }
|
||||
| { status: "no_response" }
|
||||
| { status: "failed" };
|
||||
|
||||
function postProcessResponse(text: string): string {
|
||||
let processed = text.trim();
|
||||
|
||||
// Remove <think> blocks (Qwen thinking mode artifacts)
|
||||
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
|
||||
|
||||
// Strip URLs not in allowlist
|
||||
processed = processed.replace(
|
||||
/https?:\/\/[^\s)}\]]+/gu,
|
||||
(match) => {
|
||||
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
|
||||
return match;
|
||||
}
|
||||
return "";
|
||||
},
|
||||
);
|
||||
|
||||
// Clean up double spaces from URL removal
|
||||
processed = processed.replace(/ +/gu, " ").trim();
|
||||
|
||||
// Strip markdown formatting (KakaoTalk is plaintext)
|
||||
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1"); // **bold**
|
||||
processed = processed.replace(/\*(.+?)\*/gu, "$1"); // *italic*
|
||||
processed = processed.replace(/__(.+?)__/gu, "$1"); // __underline__
|
||||
processed = processed.replace(/~~(.+?)~~/gu, "$1"); // ~~strike~~
|
||||
processed = processed.replace(/^#{1,6}\s+/gmu, ""); // ## headings
|
||||
processed = processed.replace(/^[-*]\s+/gmu, ""); // - or * list items
|
||||
processed = processed.replace(/`(.+?)`/gu, "$1"); // `inline code`
|
||||
|
||||
// Truncate to max length with Korean sentence boundary awareness
|
||||
const maxLen = getMaxResponseLength();
|
||||
if (processed.length > maxLen) {
|
||||
processed = processed.slice(0, maxLen).trimEnd();
|
||||
// Search backwards for a Korean sentence-ending pattern
|
||||
const sentenceEndPattern = /(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
|
||||
// Try progressively shorter substrings to find a sentence boundary
|
||||
for (let i = processed.length; i > maxLen * 0.5; i--) {
|
||||
const sub = processed.slice(0, i);
|
||||
if (sentenceEndPattern.test(sub)) {
|
||||
processed = sub;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return processed.trim();
|
||||
}
|
||||
|
||||
function isLowQualityResponse(
|
||||
text: string,
|
||||
options?: { allowShortResponse?: boolean },
|
||||
): boolean {
|
||||
if (!options?.allowShortResponse && text.length < 10) return true;
|
||||
// Check for excessive repetition (same phrase 3+ times)
|
||||
const words = text.split(/\s+/u);
|
||||
if (words.length >= 6) {
|
||||
const phrases = new Map<string, number>();
|
||||
for (let i = 0; i < words.length - 1; i++) {
|
||||
const bigram = `${words[i]} ${words[i + 1]}`;
|
||||
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
|
||||
}
|
||||
for (const count of phrases.values()) {
|
||||
if (count >= 3) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export class OnboardingSkill implements Skill {
|
||||
readonly name = "onboarding";
|
||||
private readonly classifier: OnboardingIntentClassifier;
|
||||
private readonly blackboardStore: OnboardingBlackboardStore;
|
||||
private readonly orchestrator: OnboardingOrchestrator;
|
||||
private readonly shadowEvalPublisher: ShadowEvalPublisher;
|
||||
private readonly tools: readonly Tool<unknown, unknown>[];
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
private readonly llmClient: LlmClient,
|
||||
private readonly roomContextService?: RoomContextService,
|
||||
shadowEvalPublisher?: ShadowEvalPublisher,
|
||||
) {
|
||||
this.classifier = new OnboardingIntentClassifier(llmClient);
|
||||
this.blackboardStore = new OnboardingBlackboardStore(garnetClient);
|
||||
this.orchestrator = new OnboardingOrchestrator(llmClient);
|
||||
this.shadowEvalPublisher =
|
||||
shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
|
||||
this.tools = createBuiltInToolRegistry().list();
|
||||
}
|
||||
|
||||
canHandle(request: AgentRequest): boolean {
|
||||
@@ -185,6 +128,15 @@ export class OnboardingSkill implements Skill {
|
||||
await this.rememberTurn(request, derived).catch((error) => {
|
||||
logger.warn({ err: error }, "[OnboardingSkill] Failed to update blackboard");
|
||||
});
|
||||
|
||||
this.shadowEvalPublisher.publish({
|
||||
classified,
|
||||
createdAt: new Date().toISOString(),
|
||||
derived,
|
||||
latencyMs: this.getCurrentLatencyMs(),
|
||||
request,
|
||||
response,
|
||||
});
|
||||
}
|
||||
|
||||
return response;
|
||||
@@ -212,11 +164,15 @@ export class OnboardingSkill implements Skill {
|
||||
derived,
|
||||
);
|
||||
|
||||
return this.generateLlmText(messages, {
|
||||
return this.generateLlmText(messages, derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
maxTokens: derived.suggestedMaxTokens,
|
||||
retryOnLowQuality:
|
||||
request.trigger === "user-message" && !derived.allowShortResponse,
|
||||
request.trigger === "user-message" &&
|
||||
!derived.allowShortResponse &&
|
||||
derived.responseMode !== "llm-with-tools",
|
||||
toolContext: this.buildToolContext(request, session),
|
||||
tools: this.selectToolsForDerived(derived),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -456,11 +412,14 @@ export class OnboardingSkill implements Skill {
|
||||
|
||||
private async generateLlmText(
|
||||
messages: ChatMessage[],
|
||||
derived: DerivedState,
|
||||
options?: {
|
||||
allowShortResponse?: boolean;
|
||||
maxTokens?: number;
|
||||
retryOnLowQuality?: boolean;
|
||||
temperature?: number;
|
||||
toolContext?: ToolContext | undefined;
|
||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||
},
|
||||
): Promise<LlmTextResult> {
|
||||
if (!this.llmClient.hasEndpoints()) {
|
||||
@@ -509,39 +468,79 @@ export class OnboardingSkill implements Skill {
|
||||
requestOptions.temperature = attempt.temperature;
|
||||
}
|
||||
|
||||
const llmResult = await this.llmClient.chatCompletion(
|
||||
const toolContext = options?.toolContext ?? {};
|
||||
const llmText = await this.orchestrator.run(
|
||||
derived,
|
||||
messages,
|
||||
requestOptions,
|
||||
{
|
||||
maxTokens: requestOptions.maxTokens,
|
||||
temperature: requestOptions.temperature,
|
||||
toolContext,
|
||||
tools: options?.tools,
|
||||
},
|
||||
);
|
||||
|
||||
if (!llmResult) {
|
||||
if (!llmText) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const processed = postProcessResponse(llmResult.content);
|
||||
if (!processed) {
|
||||
continue;
|
||||
}
|
||||
if (processed.includes("[NO_RESPONSE]")) {
|
||||
const validation = validateOnboardingResponse(
|
||||
llmText,
|
||||
derived,
|
||||
{
|
||||
allowShortResponse: options?.allowShortResponse,
|
||||
},
|
||||
);
|
||||
|
||||
if (validation.text.includes("[NO_RESPONSE]")) {
|
||||
return { status: "no_response" };
|
||||
}
|
||||
const qualityOptions =
|
||||
options?.allowShortResponse !== undefined
|
||||
? { allowShortResponse: options.allowShortResponse }
|
||||
: undefined;
|
||||
if (isLowQualityResponse(processed, qualityOptions)) {
|
||||
if (validation.status === "reject") {
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
status: "success",
|
||||
text: processed,
|
||||
text: validation.text,
|
||||
};
|
||||
}
|
||||
|
||||
return { status: "failed" };
|
||||
}
|
||||
|
||||
private buildToolContext(
|
||||
request: AgentRequest,
|
||||
session: OnboardingSession | null,
|
||||
): ToolContext {
|
||||
return {
|
||||
chatroomId: request.chatroomId,
|
||||
userId: request.userId,
|
||||
session,
|
||||
garnetClient: this.garnetClient,
|
||||
roomContextService: this.roomContextService,
|
||||
};
|
||||
}
|
||||
|
||||
private selectToolsForDerived(
|
||||
derived: DerivedState,
|
||||
): readonly Tool<unknown, unknown>[] {
|
||||
if (derived.phase.kind === "nickname-rejected") {
|
||||
return this.tools.filter((tool) =>
|
||||
tool.name === "validate-nickname" ||
|
||||
tool.name === "lookup-community-rule" ||
|
||||
tool.name === "check-onboarding-progress",
|
||||
);
|
||||
}
|
||||
|
||||
if (derived.intent.kind === "status-question") {
|
||||
return this.tools.filter((tool) =>
|
||||
tool.name === "check-onboarding-progress",
|
||||
);
|
||||
}
|
||||
|
||||
return this.tools;
|
||||
}
|
||||
|
||||
private makeTemplateResponse(
|
||||
requestId: string,
|
||||
text: string,
|
||||
@@ -574,4 +573,14 @@ export class OnboardingSkill implements Skill {
|
||||
source: "template",
|
||||
};
|
||||
}
|
||||
|
||||
private getCurrentLatencyMs(): number | undefined {
|
||||
const startedAt = getContext()?.startedAt;
|
||||
const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN;
|
||||
if (!Number.isFinite(startedTimestamp)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return Math.max(0, Date.now() - startedTimestamp);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user