Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
+8
-1
@@ -6,7 +6,11 @@ import { getLogger } from "@obs/logger";
|
||||
import { withSkillSpan, withSpan } from "@obs/spans";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { AgentRequest, AgentResponse } from "./types.js";
|
||||
import { getRequestSubject } from "../config.js";
|
||||
import {
|
||||
getAgentRequestTimeoutMs,
|
||||
getRequestSubject,
|
||||
getSafetyMarginMs,
|
||||
} from "../config.js";
|
||||
|
||||
function isValidRequest(data: unknown): data is AgentRequest {
|
||||
if (typeof data !== "object" || data === null) return false;
|
||||
@@ -64,6 +68,9 @@ export class Agent {
|
||||
await runWithContext(
|
||||
{
|
||||
chatroomId: request.chatroomId,
|
||||
deadlineAt: new Date(
|
||||
Date.now() + getAgentRequestTimeoutMs() - getSafetyMarginMs(),
|
||||
).toISOString(),
|
||||
requestId: request.requestId,
|
||||
skill: request.skill,
|
||||
startedAt: new Date().toISOString(),
|
||||
|
||||
@@ -4,6 +4,37 @@ import { getLogger } from "@obs/logger";
|
||||
|
||||
type GarnetRedisClient = ReturnType<typeof createClient>;
|
||||
|
||||
export interface StreamEntry {
|
||||
deliveriesCounter?: number | undefined;
|
||||
id: string;
|
||||
message: Record<string, string>;
|
||||
millisElapsedFromDelivery?: number | undefined;
|
||||
}
|
||||
|
||||
export interface StreamReadResult {
|
||||
messages: StreamEntry[];
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface StreamPendingEntry {
|
||||
consumer: string;
|
||||
deliveriesCounter: number;
|
||||
id: string;
|
||||
millisecondsSinceLastDelivery: number;
|
||||
}
|
||||
|
||||
interface RawStreamMessage {
|
||||
deliveriesCounter?: number | undefined;
|
||||
id: string;
|
||||
message: unknown;
|
||||
millisElapsedFromDelivery?: number | undefined;
|
||||
}
|
||||
|
||||
interface RawStreamReadResult {
|
||||
messages: RawStreamMessage[];
|
||||
name: string;
|
||||
}
|
||||
|
||||
function normalizeGarnetUrl(endpoint: string): string {
|
||||
const trimmed = endpoint.trim();
|
||||
if (!trimmed) {
|
||||
@@ -17,6 +48,29 @@ function normalizeGarnetUrl(endpoint: string): string {
|
||||
return `redis://${trimmed}`;
|
||||
}
|
||||
|
||||
function normalizeStreamFields(
|
||||
fields: unknown,
|
||||
): Record<string, string> {
|
||||
if (fields instanceof Map) {
|
||||
return Object.fromEntries(
|
||||
[...fields.entries()].map(([key, value]) => [
|
||||
String(key),
|
||||
String(value),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof fields === "object" && fields !== null) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(fields as Record<string, unknown>).map(
|
||||
([key, value]) => [key, String(value)],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
export class GarnetClient {
|
||||
private client: GarnetRedisClient | null = null;
|
||||
private readonly logger = getLogger();
|
||||
@@ -98,6 +152,27 @@ export class GarnetClient {
|
||||
await client.set(key, value);
|
||||
}
|
||||
|
||||
async setStringIfAbsent(
|
||||
key: string,
|
||||
value: string,
|
||||
ttlSeconds?: number,
|
||||
): Promise<boolean> {
|
||||
const client = this.getClient();
|
||||
const setOptions: {
|
||||
EX?: number;
|
||||
NX: true;
|
||||
} = {
|
||||
NX: true,
|
||||
};
|
||||
if (ttlSeconds && ttlSeconds > 0) {
|
||||
setOptions.EX = ttlSeconds;
|
||||
}
|
||||
const result = await client.set(key, value, {
|
||||
...setOptions,
|
||||
});
|
||||
return result === "OK";
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<number> {
|
||||
return this.getClient().del(key);
|
||||
}
|
||||
@@ -132,6 +207,163 @@ export class GarnetClient {
|
||||
return this.getClient().xAdd(key, "*", fields, trimOptions);
|
||||
}
|
||||
|
||||
async createConsumerGroup(
|
||||
key: string,
|
||||
group: string,
|
||||
startId = "$",
|
||||
): Promise<void> {
|
||||
await this.getClient().xGroupCreate(key, group, startId, {
|
||||
MKSTREAM: true,
|
||||
});
|
||||
}
|
||||
|
||||
async readGroup(
|
||||
key: string,
|
||||
group: string,
|
||||
consumer: string,
|
||||
startId: string,
|
||||
options?: {
|
||||
blockMs?: number | undefined;
|
||||
count?: number | undefined;
|
||||
},
|
||||
): Promise<StreamReadResult[] | null> {
|
||||
const readOptions: {
|
||||
BLOCK?: number;
|
||||
COUNT?: number;
|
||||
} = {};
|
||||
if (options?.blockMs !== undefined) {
|
||||
readOptions.BLOCK = options.blockMs;
|
||||
}
|
||||
if (options?.count !== undefined) {
|
||||
readOptions.COUNT = options.count;
|
||||
}
|
||||
|
||||
const reply = await this.getClient().xReadGroup(
|
||||
group,
|
||||
consumer,
|
||||
{
|
||||
key,
|
||||
id: startId,
|
||||
},
|
||||
readOptions,
|
||||
);
|
||||
|
||||
if (!reply) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (reply as RawStreamReadResult[]).map((stream) => ({
|
||||
name: String(stream.name),
|
||||
messages: stream.messages.map((message) => ({
|
||||
deliveriesCounter:
|
||||
typeof message.deliveriesCounter === "number"
|
||||
? message.deliveriesCounter
|
||||
: undefined,
|
||||
id: String(message.id),
|
||||
message: normalizeStreamFields(message.message),
|
||||
millisElapsedFromDelivery:
|
||||
typeof message.millisElapsedFromDelivery === "number"
|
||||
? message.millisElapsedFromDelivery
|
||||
: undefined,
|
||||
})),
|
||||
}));
|
||||
}
|
||||
|
||||
async acknowledgeStream(
|
||||
key: string,
|
||||
group: string,
|
||||
ids: string | string[],
|
||||
): Promise<number> {
|
||||
return this.getClient().xAck(key, group, ids);
|
||||
}
|
||||
|
||||
async getPendingStreamRange(
|
||||
key: string,
|
||||
group: string,
|
||||
count: number,
|
||||
options?: {
|
||||
consumer?: string | undefined;
|
||||
idleMs?: number | undefined;
|
||||
start?: string | undefined;
|
||||
end?: string | undefined;
|
||||
},
|
||||
): Promise<StreamPendingEntry[]> {
|
||||
const pendingOptions: {
|
||||
IDLE?: number;
|
||||
consumer?: string;
|
||||
} = {};
|
||||
if (options?.idleMs !== undefined) {
|
||||
pendingOptions.IDLE = options.idleMs;
|
||||
}
|
||||
if (options?.consumer !== undefined) {
|
||||
pendingOptions.consumer = options.consumer;
|
||||
}
|
||||
|
||||
const reply = await this.getClient().xPendingRange(
|
||||
key,
|
||||
group,
|
||||
options?.start ?? "-",
|
||||
options?.end ?? "+",
|
||||
count,
|
||||
pendingOptions,
|
||||
);
|
||||
|
||||
return reply.map((entry) => ({
|
||||
consumer: String(entry.consumer),
|
||||
deliveriesCounter: entry.deliveriesCounter,
|
||||
id: String(entry.id),
|
||||
millisecondsSinceLastDelivery: entry.millisecondsSinceLastDelivery,
|
||||
}));
|
||||
}
|
||||
|
||||
async claimPendingStream(
|
||||
key: string,
|
||||
group: string,
|
||||
consumer: string,
|
||||
minIdleTimeMs: number,
|
||||
ids: string | string[],
|
||||
options?: {
|
||||
retryCount?: number | undefined;
|
||||
},
|
||||
): Promise<StreamEntry[]> {
|
||||
const claimOptions: {
|
||||
RETRYCOUNT?: number;
|
||||
} = {};
|
||||
if (options?.retryCount !== undefined) {
|
||||
claimOptions.RETRYCOUNT = options.retryCount;
|
||||
}
|
||||
|
||||
const claimed = await this.getClient().xClaim(
|
||||
key,
|
||||
group,
|
||||
consumer,
|
||||
minIdleTimeMs,
|
||||
ids,
|
||||
claimOptions,
|
||||
);
|
||||
|
||||
return claimed.flatMap((message) => {
|
||||
if (!message) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
deliveriesCounter:
|
||||
typeof message.deliveriesCounter === "number"
|
||||
? message.deliveriesCounter
|
||||
: undefined,
|
||||
id: String(message.id),
|
||||
message: normalizeStreamFields(message.message),
|
||||
millisElapsedFromDelivery:
|
||||
typeof message.millisElapsedFromDelivery === "number"
|
||||
? message.millisElapsedFromDelivery
|
||||
: undefined,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
async scanKeys(pattern: string): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import type { EvalReport } from "@eval/types";
|
||||
|
||||
export const AGENT_REQUEST_SUBJECT = "lolympus-agent.request";
|
||||
export const AGENT_ALERT_SUBJECT = "lolympus-agent.alert";
|
||||
export const AGENT_EVAL_REPORT_SUBJECT = "lolympus-agent.eval.report";
|
||||
|
||||
export interface AlertPayload {
|
||||
kind: "latency" | "eval-score-drop" | "llm-error-rate" | "endpoint-down";
|
||||
metric: string;
|
||||
severity: "info" | "warn" | "critical";
|
||||
suggestedAction?: string | undefined;
|
||||
threshold: number;
|
||||
timestamp: string;
|
||||
traceId?: string | undefined;
|
||||
value: number;
|
||||
window: string;
|
||||
}
|
||||
|
||||
export type EvalReportPayload = EvalReport;
|
||||
+146
-2
@@ -65,8 +65,12 @@ export function getOtelSamplingRatio(): number {
|
||||
return getRatioNumber("OTEL_SAMPLING_RATIO", fallback);
|
||||
}
|
||||
|
||||
export function getOtelExporter(): "stdout" {
|
||||
return "stdout";
|
||||
export function getOtelExporter(): "stdout" | "none" {
|
||||
const env = process.env["OTEL_EXPORTER"]?.trim().toLowerCase();
|
||||
if (env === "none" || env === "stdout") {
|
||||
return env;
|
||||
}
|
||||
return getNodeEnv() === "dev" ? "none" : "stdout";
|
||||
}
|
||||
|
||||
export function getRequestSubject(): string {
|
||||
@@ -75,6 +79,10 @@ export function getRequestSubject(): string {
|
||||
);
|
||||
}
|
||||
|
||||
export function getAgentRequestTimeoutMs(): number {
|
||||
return getPositiveNumber("AGENT_REQUEST_TIMEOUT_MS", 5_000);
|
||||
}
|
||||
|
||||
export function getOnboardingSessionKeyPrefix(): string {
|
||||
return process.env["ONBOARDING_SESSION_KEY_PREFIX"]?.trim() || "lolympus";
|
||||
}
|
||||
@@ -95,6 +103,10 @@ export function getBackgroundStreamMaxLen(): number {
|
||||
return getPositiveNumber("BACKGROUND_STREAM_MAXLEN", 5_000);
|
||||
}
|
||||
|
||||
export function getEvalStreamMaxLen(): number {
|
||||
return getPositiveNumber("EVAL_STREAM_MAXLEN", 10_000);
|
||||
}
|
||||
|
||||
export function getMaxConversationMessages(): number {
|
||||
return getPositiveNumber("MAX_CONVERSATION_MESSAGES", 20);
|
||||
}
|
||||
@@ -123,14 +135,110 @@ export function getClassifierMaxTokens(): number {
|
||||
return getPositiveNumber("CLASSIFIER_MAX_TOKENS", 16);
|
||||
}
|
||||
|
||||
export function getLlmJudgeModel(): string {
|
||||
return process.env["LLM_JUDGE_MODEL"]?.trim() || "gpt-5.3-codex-spark";
|
||||
}
|
||||
|
||||
export function getLlmJudgeMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_JUDGE_MAX_TOKENS", 128);
|
||||
}
|
||||
|
||||
export function getSafetyMarginMs(): number {
|
||||
return getPositiveNumber("SAFETY_MARGIN_MS", 500);
|
||||
}
|
||||
|
||||
export function getDeadlineNearThresholdMin(): number {
|
||||
return getPositiveNumber("DEADLINE_NEAR_THRESHOLD_MIN", 30);
|
||||
}
|
||||
|
||||
export function getOpenRouterEnabled(): boolean {
|
||||
const value = process.env["OPENROUTER_ENABLED"]?.trim().toLowerCase();
|
||||
return value === "1" || value === "true" || value === "yes";
|
||||
}
|
||||
|
||||
export function getOpenRouterApiUrl(): string {
|
||||
return process.env["OPENROUTER_API_URL"]?.trim() || "https://openrouter.ai/api/v1";
|
||||
}
|
||||
|
||||
export function getTier0Model(): string {
|
||||
return (
|
||||
process.env["TIER0_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.3-codex-spark"
|
||||
);
|
||||
}
|
||||
|
||||
export function getTier1PlannerModel(): string {
|
||||
return (
|
||||
process.env["TIER1_PLANNER_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.4"
|
||||
);
|
||||
}
|
||||
|
||||
export function getTier1ResponderModel(): string {
|
||||
return (
|
||||
process.env["TIER1_RESPONDER_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.3-codex-spark"
|
||||
);
|
||||
}
|
||||
|
||||
export function getTier2CriticModel(): string {
|
||||
return (
|
||||
process.env["TIER2_CRITIC_MODEL"]?.trim() ||
|
||||
process.env["CLI_PROXY_LLM_MODEL"]?.trim() ||
|
||||
"gpt-5.4"
|
||||
);
|
||||
}
|
||||
|
||||
export function getOpenRouterTier1Model(): string {
|
||||
return (
|
||||
process.env["OPENROUTER_TIER1_MODEL"]?.trim() ||
|
||||
"openai/gpt-5.3-codex-spark"
|
||||
);
|
||||
}
|
||||
|
||||
export function getOpenRouterTier2Model(): string {
|
||||
return process.env["OPENROUTER_TIER2_MODEL"]?.trim() || "openai/gpt-5.4";
|
||||
}
|
||||
|
||||
export function getLlmMaxTokens(): number {
|
||||
return getPositiveNumber("LLM_MAX_TOKENS", 768);
|
||||
}
|
||||
|
||||
export function getTier0MaxTokens(): number {
|
||||
return getPositiveNumber("TIER0_MAX_TOKENS", 384);
|
||||
}
|
||||
|
||||
export function getTier1MaxTokens(): number {
|
||||
return getPositiveNumber("TIER1_MAX_TOKENS", 512);
|
||||
}
|
||||
|
||||
export function getTier2MaxTokens(): number {
|
||||
return getPositiveNumber("TIER2_MAX_TOKENS", 512);
|
||||
}
|
||||
|
||||
export function getPlannerMaxTokens(): number {
|
||||
return getPositiveNumber("PLANNER_MAX_TOKENS", 256);
|
||||
}
|
||||
|
||||
export function getCriticMaxTokens(): number {
|
||||
return getPositiveNumber("CRITIC_MAX_TOKENS", 512);
|
||||
}
|
||||
|
||||
export function getTier0EstimatedCallMs(): number {
|
||||
return getPositiveNumber("TIER0_ESTIMATED_CALL_MS", 600);
|
||||
}
|
||||
|
||||
export function getTier1EstimatedCallMs(): number {
|
||||
return getPositiveNumber("TIER1_ESTIMATED_CALL_MS", 1200);
|
||||
}
|
||||
|
||||
export function getTier2EstimatedCallMs(): number {
|
||||
return getPositiveNumber("TIER2_ESTIMATED_CALL_MS", 1400);
|
||||
}
|
||||
|
||||
export function getLlmTemperature(): number {
|
||||
const raw = process.env["LLM_TEMPERATURE"]?.trim();
|
||||
if (raw) {
|
||||
@@ -151,3 +259,39 @@ export function getMongoDbProjectId(): string | undefined {
|
||||
export function getRoomContextLimit(): number {
|
||||
return getPositiveNumber("ROOM_CONTEXT_LIMIT", 8);
|
||||
}
|
||||
|
||||
export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
|
||||
switch (tier) {
|
||||
case 0:
|
||||
return getRatioNumber(
|
||||
"SHADOW_EVAL_SAMPLING_RATE_TIER0",
|
||||
getNodeEnv() === "prod" ? 0.1 : 1.0,
|
||||
);
|
||||
case 1:
|
||||
return getRatioNumber(
|
||||
"SHADOW_EVAL_SAMPLING_RATE_TIER1",
|
||||
getNodeEnv() === "prod" ? 0.5 : 1.0,
|
||||
);
|
||||
case 2:
|
||||
return getRatioNumber(
|
||||
"SHADOW_EVAL_SAMPLING_RATE_TIER2",
|
||||
1.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function getAlertThresholdP95LatencyMs(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
|
||||
}
|
||||
|
||||
export function getAlertThresholdEvalScoreDropPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
|
||||
}
|
||||
|
||||
export function getAlertThresholdLlmErrorRate(): number {
|
||||
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
|
||||
}
|
||||
|
||||
export function getAlertRateLimitSec(): number {
|
||||
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { ConversationMessage } from "@agent/types";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import {
|
||||
getBackgroundStreamMaxLen,
|
||||
getConversationTtlSeconds,
|
||||
@@ -52,17 +53,24 @@ export class ConversationStore {
|
||||
await this.garnetClient.setJson(key, trimmed, getConversationTtlSeconds());
|
||||
|
||||
if (trimmed.length > 5) {
|
||||
await this.garnetClient.appendStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
{
|
||||
type: "summarize",
|
||||
chatroomId,
|
||||
userId,
|
||||
},
|
||||
{
|
||||
maxLen: getBackgroundStreamMaxLen(),
|
||||
},
|
||||
);
|
||||
void this.garnetClient
|
||||
.appendStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
{
|
||||
type: "summarize",
|
||||
chatroomId,
|
||||
userId,
|
||||
},
|
||||
{
|
||||
maxLen: getBackgroundStreamMaxLen(),
|
||||
},
|
||||
)
|
||||
.catch((error) => {
|
||||
getRequestLogger().warn(
|
||||
{ err: error, chatroomId, userId },
|
||||
"[ConversationStore] Failed to publish summarize job",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ function formatHistory(history: ConversationMessage[]): string {
|
||||
export async function summarizeConversation(
|
||||
llmClient: LlmClient,
|
||||
history: ConversationMessage[],
|
||||
existingSummary?: string | undefined,
|
||||
existingSummary?: string,
|
||||
): Promise<string> {
|
||||
if (!llmClient.hasEndpoints() || history.length === 0) {
|
||||
return existingSummary ?? "";
|
||||
|
||||
+166
-17
@@ -1,9 +1,3 @@
|
||||
function printPhaseNotice(command: string): void {
|
||||
console.info(
|
||||
`[eval] "${command}" is not implemented yet. Phase 5 will add the full eval flow.`,
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
@@ -13,26 +7,181 @@ function parsePositiveInteger(value: string | undefined): number | undefined {
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
interface ParsedArgs {
|
||||
command: string;
|
||||
flags: Map<string, string | true>;
|
||||
positionals: string[];
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): ParsedArgs {
|
||||
const [command = "run", ...rest] = argv;
|
||||
const flags = new Map<string, string | true>();
|
||||
const positionals: string[] = [];
|
||||
|
||||
for (let index = 0; index < rest.length; index++) {
|
||||
const token = rest[index];
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!token.startsWith("--")) {
|
||||
positionals.push(token);
|
||||
continue;
|
||||
}
|
||||
|
||||
const withoutPrefix = token.slice(2);
|
||||
const equalsIndex = withoutPrefix.indexOf("=");
|
||||
if (equalsIndex >= 0) {
|
||||
const key = withoutPrefix.slice(0, equalsIndex);
|
||||
const value = withoutPrefix.slice(equalsIndex + 1);
|
||||
flags.set(key, value || true);
|
||||
continue;
|
||||
}
|
||||
|
||||
const next = rest[index + 1];
|
||||
if (next && !next.startsWith("--")) {
|
||||
flags.set(withoutPrefix, next);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
flags.set(withoutPrefix, true);
|
||||
}
|
||||
|
||||
return {
|
||||
command,
|
||||
flags,
|
||||
positionals,
|
||||
};
|
||||
}
|
||||
|
||||
function getFlagString(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): string | undefined {
|
||||
const value = flags.get(key);
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function hasFlag(
|
||||
flags: Map<string, string | true>,
|
||||
key: string,
|
||||
): boolean {
|
||||
return flags.has(key);
|
||||
}
|
||||
|
||||
function printRunSummary(report: {
|
||||
generatedAt: string;
|
||||
summary: {
|
||||
cases: number;
|
||||
passRate: number;
|
||||
ruleAverageScore: number;
|
||||
averageLatencyMs: number;
|
||||
llmAverageOverall: number | null;
|
||||
llmJudgedCases: number;
|
||||
};
|
||||
baselineDelta?: {
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
averageLatencyMs: number | null;
|
||||
} | undefined;
|
||||
}, reportPath: string): void {
|
||||
const llmLine =
|
||||
report.summary.llmAverageOverall === null
|
||||
? "n/a"
|
||||
: `${report.summary.llmAverageOverall} (${report.summary.llmJudgedCases} judged)`;
|
||||
|
||||
console.info(`[eval] generatedAt=${report.generatedAt}`);
|
||||
console.info(`[eval] cases=${report.summary.cases} passRate=${report.summary.passRate}`);
|
||||
console.info(
|
||||
`[eval] ruleAverageScore=${report.summary.ruleAverageScore} averageLatencyMs=${report.summary.averageLatencyMs}`,
|
||||
);
|
||||
console.info(`[eval] llmAverageOverall=${llmLine}`);
|
||||
|
||||
if (report.baselineDelta) {
|
||||
console.info(
|
||||
`[eval] baselineDelta passRate=${report.baselineDelta.passRate ?? "n/a"} ruleAverageScore=${report.baselineDelta.ruleAverageScore ?? "n/a"} llmAverageOverall=${report.baselineDelta.llmAverageOverall ?? "n/a"} averageLatencyMs=${report.baselineDelta.averageLatencyMs ?? "n/a"}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.info(`[eval] report=${reportPath}`);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { loadEnvironmentFromNodeEnv } = await import("@common/env-loader");
|
||||
loadEnvironmentFromNodeEnv();
|
||||
|
||||
const { getLogger, initLogger } = await import("@obs/logger");
|
||||
initLogger();
|
||||
const { captureGoldenSetScaffold } = await import("./capture.js");
|
||||
const logger = getLogger();
|
||||
|
||||
const [command = "run", arg1, arg2] = process.argv.slice(2);
|
||||
const parsed = parseArgs(process.argv.slice(2));
|
||||
const { command, flags, positionals } = parsed;
|
||||
|
||||
if (command === "capture") {
|
||||
const outputPath = await captureGoldenSetScaffold({
|
||||
limit: parsePositiveInteger(arg1),
|
||||
outputPath: arg2,
|
||||
});
|
||||
getLogger().info({ outputPath }, '[eval] "capture" completed');
|
||||
return;
|
||||
try {
|
||||
if (command === "capture") {
|
||||
const { captureGoldenSetScaffold } = await import("./capture.js");
|
||||
const outputPath = await captureGoldenSetScaffold({
|
||||
limit:
|
||||
parsePositiveInteger(getFlagString(flags, "limit")) ??
|
||||
parsePositiveInteger(positionals[0]),
|
||||
outputPath: getFlagString(flags, "output") ?? positionals[1],
|
||||
});
|
||||
logger.info({ outputPath }, '[eval] "capture" completed');
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "run") {
|
||||
const { loadEvalCases, resolveEvalSetPath, runOnboardingEval } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const { publishEvalReportIfPossible } = await import(
|
||||
"./runner.js"
|
||||
);
|
||||
const setName = getFlagString(flags, "set") ?? "golden";
|
||||
const inputPath = getFlagString(flags, "input") ?? resolveEvalSetPath(setName);
|
||||
const limit = parsePositiveInteger(getFlagString(flags, "limit"));
|
||||
|
||||
if (hasFlag(flags, "dry-run")) {
|
||||
const cases = await loadEvalCases(inputPath, { limit });
|
||||
console.info(`[eval] dry-run parsed ${cases.length} cases from ${inputPath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const { report, reportPath } = await runOnboardingEval({
|
||||
enableLlmJudge: !hasFlag(flags, "skip-llm-judge"),
|
||||
inputPath,
|
||||
limit,
|
||||
outputPath: getFlagString(flags, "output"),
|
||||
setName,
|
||||
});
|
||||
const publishedReport = hasFlag(flags, "no-publish-report")
|
||||
? false
|
||||
: await publishEvalReportIfPossible(report);
|
||||
printRunSummary(report, reportPath);
|
||||
console.info(`[eval] publishedReport=${publishedReport ? "yes" : "no"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (command === "baseline") {
|
||||
const { updateEvalBaseline } = await import("./runner.js");
|
||||
const reportPath = getFlagString(flags, "report") ?? positionals[0];
|
||||
const { baseline, baselinePath, storedInRedis } = await updateEvalBaseline(reportPath);
|
||||
console.info(
|
||||
`[eval] baseline updated set=${baseline.setName} baseline=${baselinePath} redis=${storedInRedis ? "yes" : "no"}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Unknown eval command: ${command}`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{ err: error },
|
||||
`[eval] "${command}" failed`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
printPhaseNotice(command);
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -0,0 +1,3 @@
|
||||
{"id":"greeting-short","name":"Greeting should stay short and welcoming","input":{"request":{"requestId":"eval_greeting_1","skill":"onboarding","chatroomId":"room_eval_1","userId":"user_eval_1","nickname":"<nick:greeting>","trigger":"user-message","userMessage":"안녕하세요"},"session":null,"conversationHistory":[]},"expected":{"intent":"greeting","responseMode":"llm","mustIncludeAny":["환영","반갑"],"mustNotInclude":["디스코드"]}}
|
||||
{"id":"status-photo-remaining","name":"Status question should answer only the remaining step","input":{"request":{"requestId":"eval_status_1","skill":"onboarding","chatroomId":"room_eval_2","userId":"user_eval_2","nickname":"프리리/95/여/D","trigger":"user-message","userMessage":"이제 뭐 남았지"},"session":{"chatroomId":"room_eval_2","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/D","nicknameVerified":true,"photoVerified":false,"status":"pending","userId":"user_eval_2"},"conversationHistory":[{"role":"user","content":"안녕하세요","timestamp":"2026-04-16T00:00:00.000Z"},{"role":"assistant","content":"환영합니다! 반갑습니다.","timestamp":"2026-04-16T00:00:01.000Z"}]},"expected":{"intent":"status-question","responseMode":"llm","mustIncludeAny":["남았","하트"],"mustNotInclude":["디스코드"]}}
|
||||
{"id":"nickname-invalid-tier","name":"Nickname invalid flow should focus on nickname correction","input":{"request":{"requestId":"eval_nick_1","skill":"onboarding","chatroomId":"room_eval_3","userId":"user_eval_3","nickname":"프리리/95/여/다이아","trigger":"nickname-invalid"},"session":{"chatroomId":"room_eval_3","deadlineAt":"2099-01-01T00:00:00.000Z","joinedAt":"2026-04-16T00:00:00.000Z","lastEventAt":"2026-04-16T00:10:00.000Z","nickname":"프리리/95/여/다이아","nicknameVerified":false,"photoVerified":false,"status":"pending","userId":"user_eval_3"},"conversationHistory":[]},"expected":{"intent":"nickname-question","responseMode":"llm-with-tools","mustIncludeAny":["닉네임","티어","형식"],"mustNotInclude":["디스코드"]}}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { LlmClient } from "@llm/llm-client";
|
||||
import type {
|
||||
ChatCompletionOptions,
|
||||
ChatCompletionResult,
|
||||
ChatMessage,
|
||||
} from "@llm/types";
|
||||
import { judgeOnboardingWithLlm } from "./llm-judge.js";
|
||||
import type { EvalCaseDefinition, EvalDerivedSnapshot } from "../types.js";
|
||||
|
||||
class FakeLlmClient extends LlmClient {
|
||||
constructor(private readonly payload: string | null) {
|
||||
super({ endpoints: [] });
|
||||
}
|
||||
|
||||
hasEndpoints(): boolean {
|
||||
return this.payload !== null;
|
||||
}
|
||||
|
||||
chatCompletion(
|
||||
_messages: ChatMessage[],
|
||||
_options?: ChatCompletionOptions,
|
||||
): Promise<ChatCompletionResult | null> {
|
||||
if (this.payload === null) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: this.payload,
|
||||
endpoint: "http://judge.test/v1/chat/completions",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: this.payload,
|
||||
},
|
||||
provider: "cliproxy",
|
||||
raw: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const evalCase: EvalCaseDefinition = {
|
||||
id: "case-1",
|
||||
lineNumber: 1,
|
||||
input: {
|
||||
request: {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "이제 뭐 남았지",
|
||||
},
|
||||
},
|
||||
expected: {
|
||||
intent: "status-question",
|
||||
responseMode: "llm",
|
||||
},
|
||||
};
|
||||
|
||||
const derived: EvalDerivedSnapshot = {
|
||||
allowShortResponse: true,
|
||||
intent: "status-question",
|
||||
phase: "photo-pending",
|
||||
responseMode: "llm",
|
||||
shouldRespond: true,
|
||||
suggestedMaxTokens: 128,
|
||||
tier: 0,
|
||||
};
|
||||
|
||||
describe("judgeOnboardingWithLlm", () => {
|
||||
it("parses a valid JSON judge response", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient(
|
||||
JSON.stringify({
|
||||
friendliness: 5,
|
||||
accuracy: 4,
|
||||
concision: 5,
|
||||
scope: 4,
|
||||
rationale: "질문 범위에 맞게 짧고 정확하게 답했다.",
|
||||
}),
|
||||
),
|
||||
evalCase,
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
derived,
|
||||
);
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.scores.overall).toBe(4.5);
|
||||
expect(result.provider).toBe("cliproxy");
|
||||
}
|
||||
});
|
||||
|
||||
it("skips when the judge returns invalid JSON", async () => {
|
||||
const result = await judgeOnboardingWithLlm(
|
||||
new FakeLlmClient('{"friendliness":"bad"}'),
|
||||
evalCase,
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
derived,
|
||||
);
|
||||
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) {
|
||||
expect(result.reason).toBe("invalid-judge-json");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
import { z } from "zod";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLlmJudgeMaxTokens, getLlmJudgeModel } from "../../config.js";
|
||||
import type {
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
LlmJudgeResult,
|
||||
LlmJudgeScores,
|
||||
} from "../types.js";
|
||||
|
||||
const llmJudgeSchema = z.object({
|
||||
friendliness: z.number().int().min(1).max(5),
|
||||
accuracy: z.number().int().min(1).max(5),
|
||||
concision: z.number().int().min(1).max(5),
|
||||
scope: z.number().int().min(1).max(5),
|
||||
rationale: z.string().trim().min(1).max(200),
|
||||
});
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function buildJudgePrompt(
|
||||
evalCase: EvalCaseDefinition,
|
||||
responseText: string,
|
||||
derived: EvalDerivedSnapshot,
|
||||
): string {
|
||||
return [
|
||||
"다음 온보딩 에이전트 응답을 평가해 주세요.",
|
||||
"평가 축은 friendliness, accuracy, concision, scope 입니다.",
|
||||
"각 점수는 1~5의 정수만 사용하세요.",
|
||||
"scope는 사용자가 묻지 않은 추가 설명을 과하게 붙이지 않았는지 포함해 평가하세요.",
|
||||
"",
|
||||
"[request]",
|
||||
JSON.stringify(evalCase.input.request),
|
||||
"",
|
||||
"[expected]",
|
||||
JSON.stringify({
|
||||
intent: evalCase.expected.intent ?? null,
|
||||
mustIncludeAny: evalCase.expected.mustIncludeAny ?? [],
|
||||
mustNotInclude: evalCase.expected.mustNotInclude ?? [],
|
||||
responseMode: evalCase.expected.responseMode ?? null,
|
||||
}),
|
||||
"",
|
||||
"[derived]",
|
||||
JSON.stringify(derived),
|
||||
"",
|
||||
"[assistant_response]",
|
||||
responseText,
|
||||
"",
|
||||
'JSON object only: {"friendliness":1,"accuracy":1,"concision":1,"scope":1,"rationale":"..."}',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export async function judgeOnboardingWithLlm(
|
||||
llmClient: LlmClient,
|
||||
evalCase: EvalCaseDefinition,
|
||||
responseText: string,
|
||||
derived: EvalDerivedSnapshot,
|
||||
): Promise<LlmJudgeResult> {
|
||||
if (!responseText.trim()) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "empty-response",
|
||||
};
|
||||
}
|
||||
|
||||
if (!llmClient.hasEndpoints()) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "no-llm-endpoint",
|
||||
};
|
||||
}
|
||||
|
||||
const response = await llmClient.chatCompletion(
|
||||
[
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"당신은 온보딩 응답 품질 평가자입니다. 반드시 JSON object만 출력하세요.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: buildJudgePrompt(evalCase, responseText, derived),
|
||||
},
|
||||
],
|
||||
{
|
||||
jsonMode: true,
|
||||
maxTokens: getLlmJudgeMaxTokens(),
|
||||
modelOverride: getLlmJudgeModel(),
|
||||
temperature: 0,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response) {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "judge-call-failed",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = llmJudgeSchema.parse(JSON.parse(response.content));
|
||||
const scores: LlmJudgeScores = {
|
||||
friendliness: parsed.friendliness,
|
||||
accuracy: parsed.accuracy,
|
||||
concision: parsed.concision,
|
||||
scope: parsed.scope,
|
||||
overall: round(
|
||||
(parsed.friendliness +
|
||||
parsed.accuracy +
|
||||
parsed.concision +
|
||||
parsed.scope) /
|
||||
4,
|
||||
),
|
||||
};
|
||||
|
||||
return {
|
||||
skipped: false,
|
||||
model: getLlmJudgeModel(),
|
||||
endpoint: response.endpoint,
|
||||
provider: response.provider,
|
||||
scores,
|
||||
rationale: parsed.rationale,
|
||||
raw: response.content,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
skipped: true,
|
||||
reason: "invalid-judge-json",
|
||||
raw: response.content,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { judgeOnboardingRuleCase } from "./rule-judge.js";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
function makeDerived(
|
||||
overrides?: Partial<DerivedState>,
|
||||
): DerivedState {
|
||||
return {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("judgeOnboardingRuleCase", () => {
|
||||
it("passes when active checks match", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
{
|
||||
intent: "status-question",
|
||||
responseMode: "llm",
|
||||
mustIncludeAny: ["방 하트 인증"],
|
||||
mustNotInclude: ["디스코드 서버"],
|
||||
},
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.score).toBe(1);
|
||||
expect(result.validatorStatus).toBe("pass");
|
||||
});
|
||||
|
||||
it("fails intent and mustNotInclude mismatches", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
{
|
||||
intent: "help-how",
|
||||
mustNotInclude: ["디스코드"],
|
||||
},
|
||||
"디스코드 서버에 먼저 들어오시면 돼요.",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.checks.find((check) => check.name === "intent")?.passed).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
result.checks.find((check) => check.name === "mustNotInclude")?.passed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats rejecting validator results as a failed rule check", () => {
|
||||
const result = judgeOnboardingRuleCase(
|
||||
{},
|
||||
"디스코드 서버에 들어가시면 돼요.",
|
||||
makeDerived({
|
||||
intent: { kind: "photo-question" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.validatorStatus).toBe("reject");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type { DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import type { EvalCaseExpected, RuleJudgeCheck, RuleJudgeResult } from "../types.js";
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
export function judgeOnboardingRuleCase(
|
||||
expected: EvalCaseExpected,
|
||||
responseText: string,
|
||||
derived: DerivedState,
|
||||
): RuleJudgeResult {
|
||||
const validation = validateOnboardingResponse(responseText, derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
});
|
||||
const checks: RuleJudgeCheck[] = [];
|
||||
|
||||
checks.push({
|
||||
name: "validator",
|
||||
passed: validation.status !== "reject",
|
||||
skipped: false,
|
||||
detail:
|
||||
validation.reasons.length > 0
|
||||
? validation.reasons.join(", ")
|
||||
: validation.status,
|
||||
});
|
||||
|
||||
const expectedIntent = expected.intent?.trim();
|
||||
if (expectedIntent) {
|
||||
checks.push({
|
||||
name: "intent",
|
||||
passed: normalizeText(expectedIntent) === normalizeText(derived.intent.kind),
|
||||
skipped: false,
|
||||
detail: `expected=${expectedIntent}, actual=${derived.intent.kind}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "intent",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const expectedResponseMode = expected.responseMode?.trim();
|
||||
if (expectedResponseMode) {
|
||||
checks.push({
|
||||
name: "responseMode",
|
||||
passed:
|
||||
normalizeText(expectedResponseMode) ===
|
||||
normalizeText(derived.responseMode),
|
||||
skipped: false,
|
||||
detail: `expected=${expectedResponseMode}, actual=${derived.responseMode}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "responseMode",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const mustIncludeAny = expected.mustIncludeAny?.filter((item) => item.trim()) ?? [];
|
||||
if (mustIncludeAny.length > 0) {
|
||||
const normalizedResponse = normalizeText(responseText);
|
||||
const matched = mustIncludeAny.find((candidate) =>
|
||||
normalizedResponse.includes(normalizeText(candidate)),
|
||||
);
|
||||
|
||||
checks.push({
|
||||
name: "mustIncludeAny",
|
||||
passed: Boolean(matched),
|
||||
skipped: false,
|
||||
detail: matched
|
||||
? `matched=${matched}`
|
||||
: `expected one of: ${mustIncludeAny.join(", ")}`,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "mustIncludeAny",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const mustNotInclude = expected.mustNotInclude?.filter((item) => item.trim()) ?? [];
|
||||
if (mustNotInclude.length > 0) {
|
||||
const normalizedResponse = normalizeText(responseText);
|
||||
const matched = mustNotInclude.find((candidate) =>
|
||||
normalizedResponse.includes(normalizeText(candidate)),
|
||||
);
|
||||
|
||||
checks.push({
|
||||
name: "mustNotInclude",
|
||||
passed: !matched,
|
||||
skipped: false,
|
||||
detail: matched ? `forbidden=${matched}` : undefined,
|
||||
});
|
||||
} else {
|
||||
checks.push({
|
||||
name: "mustNotInclude",
|
||||
passed: true,
|
||||
skipped: true,
|
||||
});
|
||||
}
|
||||
|
||||
const activeChecks = checks.filter((check) => !check.skipped);
|
||||
const passedChecks = activeChecks.filter((check) => check.passed).length;
|
||||
|
||||
return {
|
||||
passed: activeChecks.every((check) => check.passed),
|
||||
score: activeChecks.length === 0 ? 1 : round(passedChecks / activeChecks.length),
|
||||
checks,
|
||||
validatorStatus: validation.status,
|
||||
validatorReasons: validation.reasons,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||
import { publishEvalReportIfPossible } from "./runner.js";
|
||||
import type { EvalReport } from "./types.js";
|
||||
|
||||
const natsMocks = vi.hoisted(() => ({
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
connect: vi.fn().mockResolvedValue(undefined),
|
||||
publish: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@common/nats", () => {
|
||||
return {
|
||||
NatsClient: class {
|
||||
connect = natsMocks.connect;
|
||||
publish = natsMocks.publish;
|
||||
close = natsMocks.close;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const report: EvalReport = {
|
||||
generatedAt: "2026-04-17T00:00:00.000Z",
|
||||
inputPath: "/tmp/onboarding.jsonl",
|
||||
setName: "golden",
|
||||
summary: {
|
||||
averageLatencyMs: 100,
|
||||
cases: 1,
|
||||
emptyResponses: 0,
|
||||
llmAverageAccuracy: 4,
|
||||
llmAverageConcision: 4,
|
||||
llmAverageFriendliness: 5,
|
||||
llmAverageOverall: 4.25,
|
||||
llmAverageScope: 4,
|
||||
llmJudgedCases: 1,
|
||||
passRate: 1,
|
||||
passedCases: 1,
|
||||
ruleAverageScore: 1,
|
||||
},
|
||||
cases: [],
|
||||
};
|
||||
|
||||
describe("publishEvalReportIfPossible", () => {
|
||||
it("publishes the report to the eval report subject", async () => {
|
||||
const result = await publishEvalReportIfPossible(report);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(natsMocks.connect).toHaveBeenCalledTimes(1);
|
||||
expect(natsMocks.publish).toHaveBeenCalledWith(
|
||||
AGENT_EVAL_REPORT_SUBJECT,
|
||||
report,
|
||||
);
|
||||
expect(natsMocks.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,842 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
||||
import { basename, dirname, resolve } from "node:path";
|
||||
|
||||
import type { AgentRequest } from "@agent/types";
|
||||
import { GarnetClient } from "@common/garnet";
|
||||
import { AGENT_EVAL_REPORT_SUBJECT } from "@common/nats-subjects";
|
||||
import { NatsClient } from "@common/nats";
|
||||
import { getConversationSummaryKey } from "@conv/conversation-store";
|
||||
import { LlmClient } from "@llm/llm-client";
|
||||
import type { ProviderName } from "@llm/types";
|
||||
import { runWithContext } from "@obs/context";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { RoomMessage } from "@room/types";
|
||||
import { RoomContextService } from "@room/room-context-service";
|
||||
import { OnboardingSkill } from "@skills/onboarding/index";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import { OnboardingIntentClassifier } from "@skills/onboarding/state/classifier";
|
||||
import { deriveState } from "@skills/onboarding/state/derive";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import { noopShadowEvalPublisher } from "./shadow-bus.js";
|
||||
import { getOnboardingSessionKeyPrefix, getOpenRouterApiUrl, getOpenRouterEnabled } from "../config.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import { judgeOnboardingRuleCase } from "./judges/rule-judge.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalCaseExpected,
|
||||
EvalCaseInput,
|
||||
EvalCaseResult,
|
||||
EvalClassifiedSnapshot,
|
||||
EvalDerivedSnapshot,
|
||||
EvalReport,
|
||||
EvalReportSummary,
|
||||
EvalSummaryDelta,
|
||||
} from "./types.js";
|
||||
|
||||
const DEFAULT_REPORTS_DIR = "eval-reports";
|
||||
const DEFAULT_BASELINE_FILE = `${DEFAULT_REPORTS_DIR}/baseline.json`;
|
||||
const GOLDEN_SET_PATH = "src/eval/golden/onboarding.jsonl";
|
||||
const CAPTURE_SET_PATH = "src/eval/golden/onboarding.capture.jsonl";
|
||||
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
||||
|
||||
class InMemoryGarnetClient extends GarnetClient {
|
||||
private readonly hashStore = new Map<string, Record<string, string>>();
|
||||
private readonly stringStore = new Map<string, string>();
|
||||
|
||||
getJson<T>(key: string): Promise<T | null> {
|
||||
const raw = this.stringStore.get(key);
|
||||
return Promise.resolve(raw ? (JSON.parse(raw) as T) : null);
|
||||
}
|
||||
|
||||
setJson(
|
||||
key: string,
|
||||
value: unknown,
|
||||
_ttlSeconds?: number,
|
||||
): Promise<void> {
|
||||
this.stringStore.set(key, JSON.stringify(value));
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
getString(key: string): Promise<string | null> {
|
||||
return Promise.resolve(this.stringStore.get(key) ?? null);
|
||||
}
|
||||
|
||||
setString(
|
||||
key: string,
|
||||
value: string,
|
||||
_ttlSeconds?: number,
|
||||
): Promise<void> {
|
||||
this.stringStore.set(key, value);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
delete(key: string): Promise<number> {
|
||||
const stringDeleted = this.stringStore.delete(key);
|
||||
const hashDeleted = this.hashStore.delete(key);
|
||||
return Promise.resolve(stringDeleted || hashDeleted ? 1 : 0);
|
||||
}
|
||||
|
||||
getHash(key: string): Promise<Record<string, string>> {
|
||||
return Promise.resolve({ ...(this.hashStore.get(key) ?? {}) });
|
||||
}
|
||||
|
||||
setHash(
|
||||
key: string,
|
||||
fields: Record<string, string>,
|
||||
): Promise<void> {
|
||||
this.hashStore.set(key, { ...fields });
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class InMemoryRoomContextService extends RoomContextService {
|
||||
constructor(private readonly messages: RoomMessage[]) {
|
||||
super();
|
||||
}
|
||||
|
||||
getRecentMessages(
|
||||
_chatroomId: string,
|
||||
limit?: number,
|
||||
): Promise<RoomMessage[]> {
|
||||
const effectiveLimit = limit ?? this.messages.length;
|
||||
return Promise.resolve(this.messages.slice(-effectiveLimit));
|
||||
}
|
||||
}
|
||||
|
||||
interface RawEvalCaseLine {
|
||||
expected?: Record<string, unknown>;
|
||||
id?: unknown;
|
||||
input?: Record<string, unknown>;
|
||||
name?: unknown;
|
||||
notes?: unknown;
|
||||
}
|
||||
|
||||
export interface EvalRunOptions {
|
||||
enableLlmJudge?: boolean | undefined;
|
||||
inputPath?: string | undefined;
|
||||
limit?: number | undefined;
|
||||
outputPath?: string | undefined;
|
||||
setName?: string | undefined;
|
||||
}
|
||||
|
||||
function round(value: number): number {
|
||||
return Number(value.toFixed(3));
|
||||
}
|
||||
|
||||
function average(values: number[]): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return round(values.reduce((sum, value) => sum + value, 0) / values.length);
|
||||
}
|
||||
|
||||
function averageOrZero(values: number[]): number {
|
||||
return values.length === 0 ? 0 : round(values.reduce((sum, value) => sum + value, 0) / values.length);
|
||||
}
|
||||
|
||||
function firstNonEmpty(
|
||||
...values: Array<string | null | undefined>
|
||||
): string | undefined {
|
||||
for (const value of values) {
|
||||
if (value?.trim()) {
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseEndpointList(
|
||||
...values: Array<string | null | undefined>
|
||||
): string[] {
|
||||
const endpoints: string[] = [];
|
||||
|
||||
for (const value of values) {
|
||||
if (!value?.trim()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const item of value.split(/[\n,]/u)) {
|
||||
const trimmed = item.trim();
|
||||
if (trimmed) {
|
||||
endpoints.push(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...new Set(endpoints)];
|
||||
}
|
||||
|
||||
function toProviderEndpoints(
|
||||
urls: string[],
|
||||
provider?: ProviderName,
|
||||
): Array<string | { url: string; provider?: ProviderName | undefined }> {
|
||||
if (!provider) {
|
||||
return urls;
|
||||
}
|
||||
|
||||
return urls.map((url) => ({ url, provider }));
|
||||
}
|
||||
|
||||
function createEvalLlmClient(): LlmClient {
|
||||
const cliproxyEndpoints = parseEndpointList(process.env["CLI_PROXY_API_URLS"]);
|
||||
const llamaEndpoints = parseEndpointList(
|
||||
process.env["AGENT_LLM_API_URLS"],
|
||||
process.env["LLAMA_SERVER_API_URL"],
|
||||
);
|
||||
const openRouterEndpoints = getOpenRouterEnabled()
|
||||
? parseEndpointList(process.env["OPENROUTER_API_URL"], getOpenRouterApiUrl())
|
||||
: [];
|
||||
|
||||
const llmApiKey = firstNonEmpty(
|
||||
process.env["AGENT_LLM_API_KEY"],
|
||||
process.env["LLAMA_SERVER_API_KEY"],
|
||||
);
|
||||
const providerApiKeys = Object.fromEntries(
|
||||
[
|
||||
["cliproxy", firstNonEmpty(process.env["CLI_PROXY_API_KEY"])],
|
||||
["llama", llmApiKey],
|
||||
["openrouter", firstNonEmpty(process.env["OPENROUTER_API_KEY"])],
|
||||
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
||||
) as Partial<Record<ProviderName, string>>;
|
||||
|
||||
return new LlmClient({
|
||||
endpoints: [
|
||||
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
|
||||
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
|
||||
...toProviderEndpoints(llamaEndpoints, "llama"),
|
||||
],
|
||||
apiKey: llmApiKey,
|
||||
model: firstNonEmpty(process.env["AGENT_LLM_MODEL"]),
|
||||
providerApiKeys,
|
||||
});
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
function getStringField(record: Record<string, unknown>, key: string): string | undefined {
|
||||
const value = record[key];
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function getStringArrayField(
|
||||
record: Record<string, unknown>,
|
||||
key: string,
|
||||
): string[] | undefined {
|
||||
const value = record[key];
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const strings = value.filter((item): item is string => typeof item === "string");
|
||||
return strings.length > 0 ? strings : [];
|
||||
}
|
||||
|
||||
function toAgentRequest(raw: Record<string, unknown>): AgentRequest {
|
||||
const requestId = getStringField(raw, "requestId");
|
||||
const skill = getStringField(raw, "skill");
|
||||
const chatroomId = getStringField(raw, "chatroomId");
|
||||
const userId = getStringField(raw, "userId");
|
||||
const nickname = getStringField(raw, "nickname");
|
||||
const trigger = getStringField(raw, "trigger");
|
||||
|
||||
if (!requestId || !skill || !chatroomId || !userId || !nickname || !trigger) {
|
||||
throw new Error("input.request is missing required AgentRequest fields");
|
||||
}
|
||||
|
||||
const request: AgentRequest = {
|
||||
requestId,
|
||||
skill,
|
||||
chatroomId,
|
||||
userId,
|
||||
nickname,
|
||||
trigger,
|
||||
};
|
||||
|
||||
const userMessage = getStringField(raw, "userMessage");
|
||||
if (userMessage !== undefined) {
|
||||
request.userMessage = userMessage;
|
||||
}
|
||||
|
||||
const context = raw["context"];
|
||||
if (isRecord(context)) {
|
||||
request.context = context;
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
function parseConversationHistory(raw: unknown): EvalCaseInput["conversationHistory"] {
|
||||
if (!Array.isArray(raw)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return raw
|
||||
.filter((item): item is Record<string, unknown> => isRecord(item))
|
||||
.map((item) => {
|
||||
const role = item["role"];
|
||||
const content = getStringField(item, "content") ?? "";
|
||||
const timestamp = getStringField(item, "timestamp") ?? new Date(0).toISOString();
|
||||
|
||||
if (role !== "user" && role !== "assistant") {
|
||||
throw new Error("conversationHistory contains an invalid role");
|
||||
}
|
||||
|
||||
return {
|
||||
role,
|
||||
content,
|
||||
timestamp,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function parseRoomContext(raw: unknown): EvalCaseInput["roomContext"] {
|
||||
if (!Array.isArray(raw)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return raw
|
||||
.filter((item): item is Record<string, unknown> => isRecord(item))
|
||||
.map((item) => ({
|
||||
senderId: getStringField(item, "senderId") ?? "",
|
||||
nickname: getStringField(item, "nickname") ?? "알 수 없음",
|
||||
message: getStringField(item, "message") ?? "",
|
||||
timestamp: getStringField(item, "timestamp") ?? new Date(0).toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
function parseExpected(raw: Record<string, unknown> | undefined): EvalCaseExpected {
|
||||
if (!raw) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const expected: EvalCaseExpected = {};
|
||||
const intent = getStringField(raw, "intent");
|
||||
const responseMode = getStringField(raw, "responseMode");
|
||||
const mustIncludeAny = getStringArrayField(raw, "mustIncludeAny");
|
||||
const mustNotInclude = getStringArrayField(raw, "mustNotInclude");
|
||||
|
||||
if (intent) {
|
||||
expected.intent = intent;
|
||||
}
|
||||
if (responseMode) {
|
||||
expected.responseMode = responseMode;
|
||||
}
|
||||
if (mustIncludeAny) {
|
||||
expected.mustIncludeAny = mustIncludeAny;
|
||||
}
|
||||
if (mustNotInclude) {
|
||||
expected.mustNotInclude = mustNotInclude;
|
||||
}
|
||||
|
||||
return expected;
|
||||
}
|
||||
|
||||
function parseEvalCaseLine(
|
||||
raw: RawEvalCaseLine,
|
||||
lineNumber: number,
|
||||
sourcePath: string,
|
||||
): EvalCaseDefinition {
|
||||
if (!isRecord(raw.input)) {
|
||||
throw new Error("input is required");
|
||||
}
|
||||
|
||||
const requestRaw = raw.input["request"];
|
||||
if (!isRecord(requestRaw)) {
|
||||
throw new Error("input.request is required");
|
||||
}
|
||||
|
||||
const input: EvalCaseInput = {
|
||||
request: toAgentRequest(requestRaw),
|
||||
};
|
||||
|
||||
const conversationHistory = parseConversationHistory(raw.input["conversationHistory"]);
|
||||
if (conversationHistory) {
|
||||
input.conversationHistory = conversationHistory;
|
||||
}
|
||||
|
||||
const roomContext = parseRoomContext(raw.input["roomContext"]);
|
||||
if (roomContext) {
|
||||
input.roomContext = roomContext;
|
||||
}
|
||||
|
||||
if (raw.input["summary"] !== undefined) {
|
||||
const summary = getStringField(raw.input, "summary");
|
||||
if (summary !== undefined) {
|
||||
input.summary = summary;
|
||||
}
|
||||
}
|
||||
|
||||
if (raw.input["session"] === null || isRecord(raw.input["session"])) {
|
||||
input.session = (raw.input["session"] ?? null) as OnboardingSession | null;
|
||||
}
|
||||
|
||||
if (raw.input["blackboard"] === null || isRecord(raw.input["blackboard"])) {
|
||||
input.blackboard = (raw.input["blackboard"] ?? null) as Blackboard | null;
|
||||
}
|
||||
|
||||
return {
|
||||
id:
|
||||
typeof raw.id === "string" && raw.id.trim()
|
||||
? raw.id.trim()
|
||||
: `${basename(sourcePath)}:${lineNumber}`,
|
||||
lineNumber,
|
||||
name: typeof raw.name === "string" ? raw.name : undefined,
|
||||
notes: typeof raw.notes === "string" ? raw.notes : undefined,
|
||||
input,
|
||||
expected: parseExpected(isRecord(raw.expected) ? raw.expected : undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
||||
tier: derived.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function toClassifiedSnapshot(classified: ClassifiedIntent): EvalClassifiedSnapshot {
|
||||
return {
|
||||
confidence: round(classified.confidence),
|
||||
intent: classified.intent.kind,
|
||||
source: classified.source,
|
||||
};
|
||||
}
|
||||
|
||||
function toRoomMessages(
|
||||
input: EvalCaseInput["roomContext"],
|
||||
): RoomMessage[] {
|
||||
return (
|
||||
input?.map((message) => ({
|
||||
senderId: message.senderId,
|
||||
nickname: message.nickname,
|
||||
message: message.message,
|
||||
timestamp: new Date(message.timestamp),
|
||||
})) ?? []
|
||||
);
|
||||
}
|
||||
|
||||
async function hydrateEvalState(
|
||||
garnetClient: InMemoryGarnetClient,
|
||||
input: EvalCaseInput,
|
||||
): Promise<void> {
|
||||
const {
|
||||
request: { chatroomId, userId },
|
||||
} = input;
|
||||
|
||||
if (input.session !== undefined) {
|
||||
await garnetClient.setJson(
|
||||
`${getOnboardingSessionKeyPrefix()}:onboarding:${chatroomId}:${userId}`,
|
||||
input.session,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.blackboard !== undefined) {
|
||||
await garnetClient.setJson(
|
||||
`lolympus-agent:bb:${chatroomId}:${userId}`,
|
||||
input.blackboard,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.summary?.trim()) {
|
||||
await garnetClient.setString(
|
||||
getConversationSummaryKey(chatroomId, userId),
|
||||
input.summary,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function createSummary(results: EvalCaseResult[]): EvalReportSummary {
|
||||
const llmResults = results.flatMap((result) =>
|
||||
result.llmJudge.skipped ? [] : [result.llmJudge],
|
||||
);
|
||||
|
||||
return {
|
||||
averageLatencyMs: averageOrZero(results.map((result) => result.latencyMs)),
|
||||
cases: results.length,
|
||||
emptyResponses: results.filter((result) => !result.response.text.trim()).length,
|
||||
llmAverageAccuracy: average(
|
||||
llmResults.map((result) => result.scores.accuracy),
|
||||
),
|
||||
llmAverageConcision: average(
|
||||
llmResults.map((result) => result.scores.concision),
|
||||
),
|
||||
llmAverageFriendliness: average(
|
||||
llmResults.map((result) => result.scores.friendliness),
|
||||
),
|
||||
llmAverageOverall: average(
|
||||
llmResults.map((result) => result.scores.overall),
|
||||
),
|
||||
llmAverageScope: average(
|
||||
llmResults.map((result) => result.scores.scope),
|
||||
),
|
||||
llmJudgedCases: llmResults.length,
|
||||
passRate:
|
||||
results.length === 0
|
||||
? 0
|
||||
: round(
|
||||
results.filter((result) => result.ruleJudge.passed).length /
|
||||
results.length,
|
||||
),
|
||||
passedCases: results.filter((result) => result.ruleJudge.passed).length,
|
||||
ruleAverageScore: averageOrZero(
|
||||
results.map((result) => result.ruleJudge.score),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function createBaselineDelta(
|
||||
current: EvalReportSummary,
|
||||
baseline: EvalReportSummary,
|
||||
): EvalSummaryDelta {
|
||||
const subtract = (
|
||||
left: number | null,
|
||||
right: number | null,
|
||||
): number | null => {
|
||||
if (left === null || right === null) {
|
||||
return null;
|
||||
}
|
||||
return round(left - right);
|
||||
};
|
||||
|
||||
return {
|
||||
averageLatencyMs: subtract(current.averageLatencyMs, baseline.averageLatencyMs),
|
||||
llmAverageOverall: subtract(
|
||||
current.llmAverageOverall,
|
||||
baseline.llmAverageOverall,
|
||||
),
|
||||
passRate: subtract(current.passRate, baseline.passRate),
|
||||
ruleAverageScore: subtract(current.ruleAverageScore, baseline.ruleAverageScore),
|
||||
};
|
||||
}
|
||||
|
||||
function buildDefaultReportPath(timestamp: string): string {
|
||||
const filename = `${timestamp.replace(/[:.]/gu, "-")}.json`;
|
||||
return resolve(process.cwd(), DEFAULT_REPORTS_DIR, filename);
|
||||
}
|
||||
|
||||
async function writeReport(
|
||||
report: EvalReport,
|
||||
outputPath: string,
|
||||
): Promise<string> {
|
||||
const resolved = resolve(process.cwd(), outputPath);
|
||||
await mkdir(dirname(resolved), { recursive: true });
|
||||
await writeFile(resolved, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function loadBaselineFile(): Promise<EvalBaselineFile | null> {
|
||||
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
|
||||
if (!existsSync(baselinePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const raw = await readFile(baselinePath, "utf8");
|
||||
return JSON.parse(raw) as EvalBaselineFile;
|
||||
}
|
||||
|
||||
export function resolveEvalSetPath(setNameOrPath = "golden"): string {
|
||||
if (setNameOrPath === "golden") {
|
||||
return resolve(process.cwd(), GOLDEN_SET_PATH);
|
||||
}
|
||||
|
||||
if (setNameOrPath === "capture") {
|
||||
return resolve(process.cwd(), CAPTURE_SET_PATH);
|
||||
}
|
||||
|
||||
if (setNameOrPath.endsWith(".jsonl") || setNameOrPath.includes("/")) {
|
||||
return resolve(process.cwd(), setNameOrPath);
|
||||
}
|
||||
|
||||
return resolve(process.cwd(), "src/eval/golden", `${setNameOrPath}.jsonl`);
|
||||
}
|
||||
|
||||
export async function loadEvalCases(
|
||||
inputPath: string,
|
||||
options?: { limit?: number | undefined },
|
||||
): Promise<EvalCaseDefinition[]> {
|
||||
const resolvedPath = resolve(process.cwd(), inputPath);
|
||||
if (!existsSync(resolvedPath)) {
|
||||
throw new Error(`Eval set not found: ${resolvedPath}`);
|
||||
}
|
||||
|
||||
const raw = await readFile(resolvedPath, "utf8");
|
||||
const lines = raw.split(/\r?\n/u).filter((line) => line.trim());
|
||||
const cases = lines.map((line, index) =>
|
||||
parseEvalCaseLine(
|
||||
JSON.parse(line) as RawEvalCaseLine,
|
||||
index + 1,
|
||||
resolvedPath,
|
||||
),
|
||||
);
|
||||
|
||||
return options?.limit ? cases.slice(0, options.limit) : cases;
|
||||
}
|
||||
|
||||
async function evaluateCase(
|
||||
evalCase: EvalCaseDefinition,
|
||||
llmClient: LlmClient,
|
||||
enableLlmJudge: boolean,
|
||||
): Promise<EvalCaseResult> {
|
||||
const input = evalCase.input;
|
||||
const inMemoryGarnet = new InMemoryGarnetClient();
|
||||
await hydrateEvalState(inMemoryGarnet, input);
|
||||
|
||||
const roomContextService =
|
||||
input.roomContext && input.roomContext.length > 0
|
||||
? new InMemoryRoomContextService(toRoomMessages(input.roomContext))
|
||||
: undefined;
|
||||
const skill = new OnboardingSkill(
|
||||
inMemoryGarnet,
|
||||
llmClient,
|
||||
roomContextService,
|
||||
noopShadowEvalPublisher,
|
||||
);
|
||||
const classifier = new OnboardingIntentClassifier(llmClient);
|
||||
|
||||
return runWithContext(
|
||||
{
|
||||
chatroomId: input.request.chatroomId,
|
||||
deadlineAt: new Date(Date.now() + 4_500).toISOString(),
|
||||
requestId: input.request.requestId,
|
||||
skill: input.request.skill,
|
||||
startedAt: new Date().toISOString(),
|
||||
trigger: input.request.trigger,
|
||||
userId: input.request.userId,
|
||||
},
|
||||
async () => {
|
||||
const classified = await classifier.classify(
|
||||
input.request.userMessage,
|
||||
input.request.trigger,
|
||||
);
|
||||
const derived = deriveState(
|
||||
input.session ?? null,
|
||||
input.request.trigger,
|
||||
classified.intent,
|
||||
);
|
||||
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const response = await skill.handle(
|
||||
input.request,
|
||||
input.conversationHistory ?? [],
|
||||
);
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const derivedSnapshot = toDerivedSnapshot(derived);
|
||||
const ruleJudge = judgeOnboardingRuleCase(
|
||||
evalCase.expected,
|
||||
response.text,
|
||||
derived,
|
||||
);
|
||||
const llmJudge = enableLlmJudge
|
||||
? await judgeOnboardingWithLlm(
|
||||
llmClient,
|
||||
evalCase,
|
||||
response.text,
|
||||
derivedSnapshot,
|
||||
)
|
||||
: {
|
||||
skipped: true as const,
|
||||
reason: "disabled",
|
||||
};
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
name: evalCase.name,
|
||||
latencyMs,
|
||||
classified: toClassifiedSnapshot(classified),
|
||||
derived: derivedSnapshot,
|
||||
request: input.request,
|
||||
response,
|
||||
ruleJudge,
|
||||
llmJudge,
|
||||
};
|
||||
} catch (error) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
|
||||
return {
|
||||
id: evalCase.id,
|
||||
lineNumber: evalCase.lineNumber,
|
||||
name: evalCase.name,
|
||||
latencyMs,
|
||||
classified: toClassifiedSnapshot(classified),
|
||||
derived: toDerivedSnapshot(derived),
|
||||
request: input.request,
|
||||
response: {
|
||||
requestId: input.request.requestId,
|
||||
text: "",
|
||||
confidence: 0,
|
||||
source: "template",
|
||||
metadata: {
|
||||
error: errorMessage,
|
||||
},
|
||||
},
|
||||
ruleJudge: judgeOnboardingRuleCase(evalCase.expected, "", derived),
|
||||
llmJudge: {
|
||||
skipped: true,
|
||||
reason: "skill-error",
|
||||
},
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export async function runOnboardingEval(
|
||||
options?: EvalRunOptions,
|
||||
): Promise<{ report: EvalReport; reportPath: string }> {
|
||||
const logger = getLogger();
|
||||
const setName = options?.setName ?? "golden";
|
||||
const inputPath = options?.inputPath
|
||||
? resolve(process.cwd(), options.inputPath)
|
||||
: resolveEvalSetPath(setName);
|
||||
const evalCases = await loadEvalCases(inputPath, {
|
||||
limit: options?.limit,
|
||||
});
|
||||
|
||||
const llmClient = createEvalLlmClient();
|
||||
const enableLlmJudge = options?.enableLlmJudge ?? true;
|
||||
const caseResults: EvalCaseResult[] = [];
|
||||
|
||||
for (const evalCase of evalCases) {
|
||||
caseResults.push(await evaluateCase(evalCase, llmClient, enableLlmJudge));
|
||||
}
|
||||
|
||||
const summary = createSummary(caseResults);
|
||||
const baseline = await loadBaselineFile();
|
||||
const report: EvalReport = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
inputPath,
|
||||
setName,
|
||||
summary,
|
||||
cases: caseResults,
|
||||
};
|
||||
|
||||
if (baseline) {
|
||||
report.baselineDelta = createBaselineDelta(summary, baseline.summary);
|
||||
}
|
||||
|
||||
const reportPath = await writeReport(
|
||||
report,
|
||||
options?.outputPath ?? buildDefaultReportPath(report.generatedAt),
|
||||
);
|
||||
logger.info(
|
||||
{
|
||||
cases: summary.cases,
|
||||
passRate: summary.passRate,
|
||||
reportPath,
|
||||
},
|
||||
"[Eval] run completed",
|
||||
);
|
||||
|
||||
return { report, reportPath };
|
||||
}
|
||||
|
||||
export async function findLatestEvalReportPath(): Promise<string | null> {
|
||||
const reportsDir = resolve(process.cwd(), DEFAULT_REPORTS_DIR);
|
||||
if (!existsSync(reportsDir)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const entries = await readdir(reportsDir, { withFileTypes: true });
|
||||
const candidates = entries
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => entry.name)
|
||||
.filter((name) => name.endsWith(".json") && name !== "baseline.json")
|
||||
.sort((left, right) => right.localeCompare(left));
|
||||
|
||||
if (candidates.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const latest = candidates[0];
|
||||
if (!latest) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return resolve(reportsDir, latest);
|
||||
}
|
||||
|
||||
export async function updateEvalBaseline(
|
||||
reportPath?: string,
|
||||
): Promise<{
|
||||
baseline: EvalBaselineFile;
|
||||
baselinePath: string;
|
||||
storedInRedis: boolean;
|
||||
}> {
|
||||
const sourcePath =
|
||||
reportPath?.trim() ? resolve(process.cwd(), reportPath) : await findLatestEvalReportPath();
|
||||
if (!sourcePath) {
|
||||
throw new Error("No eval report found to baseline");
|
||||
}
|
||||
|
||||
const raw = await readFile(sourcePath, "utf8");
|
||||
const report = JSON.parse(raw) as EvalReport;
|
||||
const baseline: EvalBaselineFile = {
|
||||
reportPath: sourcePath,
|
||||
setName: report.setName,
|
||||
summary: report.summary,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const baselinePath = resolve(process.cwd(), DEFAULT_BASELINE_FILE);
|
||||
await mkdir(dirname(baselinePath), { recursive: true });
|
||||
await writeFile(baselinePath, `${JSON.stringify(baseline, null, 2)}\n`, "utf8");
|
||||
|
||||
const storedInRedis = await persistBaselineInRedis(baseline);
|
||||
|
||||
return { baseline, baselinePath, storedInRedis };
|
||||
}
|
||||
|
||||
async function persistBaselineInRedis(
|
||||
baseline: EvalBaselineFile,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const { GarnetClient } = await import("@common/garnet");
|
||||
const garnetClient = new GarnetClient();
|
||||
await garnetClient.connect();
|
||||
try {
|
||||
await garnetClient.setString(
|
||||
EVAL_BASELINE_KEY,
|
||||
JSON.stringify(baseline),
|
||||
);
|
||||
return true;
|
||||
} finally {
|
||||
await garnetClient.close();
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function publishEvalReportIfPossible(
|
||||
report: EvalReport,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const natsClient = new NatsClient();
|
||||
await natsClient.connect();
|
||||
try {
|
||||
await natsClient.publish(AGENT_EVAL_REPORT_SUBJECT, report);
|
||||
return true;
|
||||
} finally {
|
||||
await natsClient.close();
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
ShadowEvalBus,
|
||||
shouldSampleShadowEval,
|
||||
} from "./shadow-bus.js";
|
||||
|
||||
const request: AgentRequest = {
|
||||
requestId: "req-1",
|
||||
skill: "onboarding",
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
nickname: "프리리/95/여/D",
|
||||
trigger: "user-message",
|
||||
userMessage: "이제 뭐 남았지",
|
||||
};
|
||||
|
||||
const response: AgentResponse = {
|
||||
requestId: "req-1",
|
||||
text: "이제 방 하트 인증만 남았어요.",
|
||||
confidence: 0.9,
|
||||
source: "llm",
|
||||
};
|
||||
|
||||
const classified: ClassifiedIntent = {
|
||||
intent: { kind: "status-question" },
|
||||
confidence: 0.98,
|
||||
source: "regex",
|
||||
};
|
||||
|
||||
const derived: DerivedState = {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "status-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 128,
|
||||
allowShortResponse: true,
|
||||
tier: 0,
|
||||
};
|
||||
|
||||
describe("shouldSampleShadowEval", () => {
|
||||
it("samples when the random value is under the tier rate", () => {
|
||||
expect(shouldSampleShadowEval(0, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not sample when the random value exceeds the tier rate", () => {
|
||||
expect(shouldSampleShadowEval(0, 1)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ShadowEvalBus", () => {
|
||||
it("publishes a shadow-eval stream entry when sampled", () => {
|
||||
const appendStream = vi.fn().mockResolvedValue("1-0");
|
||||
const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0);
|
||||
|
||||
new ShadowEvalBus({ appendStream }).publish({
|
||||
classified,
|
||||
createdAt: "2026-04-16T00:00:00.000Z",
|
||||
derived,
|
||||
request,
|
||||
response,
|
||||
});
|
||||
|
||||
expect(appendStream).toHaveBeenCalledTimes(1);
|
||||
expect(appendStream.mock.calls[0]?.[0]).toBe("lolympus-agent:background");
|
||||
expect(appendStream.mock.calls[0]?.[1]).toMatchObject({
|
||||
type: "shadow-eval",
|
||||
requestId: "req-1",
|
||||
tier: "0",
|
||||
});
|
||||
|
||||
randomSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("skips publishing for empty responses", () => {
|
||||
const appendStream = vi.fn().mockResolvedValue("1-0");
|
||||
|
||||
new ShadowEvalBus({ appendStream }).publish({
|
||||
classified,
|
||||
createdAt: "2026-04-16T00:00:00.000Z",
|
||||
derived,
|
||||
request,
|
||||
response: {
|
||||
...response,
|
||||
text: "",
|
||||
},
|
||||
});
|
||||
|
||||
expect(appendStream).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import {
|
||||
getBackgroundStreamMaxLen,
|
||||
getShadowEvalSamplingRate,
|
||||
} from "../config.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
|
||||
export interface ShadowEvalEnvelope {
|
||||
classified: ClassifiedIntent;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
|
||||
export interface ShadowEvalPublisher {
|
||||
publish(envelope: ShadowEvalEnvelope): void;
|
||||
}
|
||||
|
||||
export const noopShadowEvalPublisher: ShadowEvalPublisher = {
|
||||
publish(): void {},
|
||||
};
|
||||
|
||||
export function shouldSampleShadowEval(
|
||||
tier: 0 | 1 | 2,
|
||||
randomValue = Math.random(),
|
||||
): boolean {
|
||||
return randomValue < getShadowEvalSamplingRate(tier);
|
||||
}
|
||||
|
||||
export class ShadowEvalBus implements ShadowEvalPublisher {
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "appendStream">,
|
||||
) {}
|
||||
|
||||
publish(envelope: ShadowEvalEnvelope): void {
|
||||
if (!envelope.response.text.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!shouldSampleShadowEval(envelope.derived.tier)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const logger = getRequestLogger();
|
||||
|
||||
void this.garnetClient
|
||||
.appendStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval",
|
||||
requestId: envelope.request.requestId,
|
||||
chatroomId: envelope.request.chatroomId,
|
||||
userId: envelope.request.userId,
|
||||
tier: String(envelope.derived.tier),
|
||||
latencyMs:
|
||||
envelope.latencyMs !== undefined
|
||||
? String(envelope.latencyMs)
|
||||
: "",
|
||||
input: JSON.stringify({
|
||||
request: envelope.request,
|
||||
}),
|
||||
output: JSON.stringify(envelope.response),
|
||||
derived: JSON.stringify(envelope.derived),
|
||||
classified: JSON.stringify(envelope.classified),
|
||||
createdAt: envelope.createdAt,
|
||||
},
|
||||
{
|
||||
maxLen: getBackgroundStreamMaxLen(),
|
||||
},
|
||||
)
|
||||
.catch((error) => {
|
||||
logger.warn(
|
||||
{
|
||||
err: error,
|
||||
requestId: envelope.request.requestId,
|
||||
},
|
||||
"[ShadowEvalBus] Failed to publish shadow eval job",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
calculateEvalScoreDropPct,
|
||||
calculateLlmErrorRate,
|
||||
calculatePercentile,
|
||||
} from "./shadow-worker.js";
|
||||
|
||||
describe("calculatePercentile", () => {
|
||||
it("returns null for empty input", () => {
|
||||
expect(calculatePercentile([], 0.95)).toBeNull();
|
||||
});
|
||||
|
||||
it("calculates the p95 from sorted values", () => {
|
||||
expect(calculatePercentile([100, 200, 300, 400, 500], 0.95)).toBe(500);
|
||||
});
|
||||
});
|
||||
|
||||
describe("calculateEvalScoreDropPct", () => {
|
||||
it("returns null without a valid baseline", () => {
|
||||
expect(calculateEvalScoreDropPct(4.2, null)).toBeNull();
|
||||
expect(calculateEvalScoreDropPct(4.2, 0)).toBeNull();
|
||||
});
|
||||
|
||||
it("computes percentage drop from baseline", () => {
|
||||
expect(calculateEvalScoreDropPct(3.6, 4.0)).toBeCloseTo(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe("calculateLlmErrorRate", () => {
|
||||
it("returns null when no observations are present", () => {
|
||||
expect(calculateLlmErrorRate([])).toBeNull();
|
||||
});
|
||||
|
||||
it("computes error ratio across observations", () => {
|
||||
expect(
|
||||
calculateLlmErrorRate([
|
||||
{
|
||||
llmJudgeErrored: true,
|
||||
llmOverall: null,
|
||||
recordedAt: 1,
|
||||
responseLatencyMs: 100,
|
||||
},
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4.2,
|
||||
recordedAt: 2,
|
||||
responseLatencyMs: 200,
|
||||
},
|
||||
]),
|
||||
).toBe(0.5);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,741 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type {
|
||||
GarnetClient,
|
||||
StreamEntry,
|
||||
StreamPendingEntry,
|
||||
} from "@common/garnet";
|
||||
import { summarizeConversation } from "@conv/summarizer";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { AlertSender } from "@obs/alert";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import {
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getEvalStreamMaxLen,
|
||||
} from "../config.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
} from "./types.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
|
||||
const BACKGROUND_GROUP = "bg-worker";
|
||||
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
||||
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
||||
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
|
||||
const ALERT_WINDOW_MS = 5 * 60_000;
|
||||
const ALERT_WINDOW_LABEL = "5m";
|
||||
const ALERT_TICK_MS = 60_000;
|
||||
const MAX_DELIVERIES = 3;
|
||||
const MIN_ERROR_RATE_OBSERVATIONS = 5;
|
||||
const MIN_LATENCY_OBSERVATIONS = 5;
|
||||
const MIN_SCORE_OBSERVATIONS = 3;
|
||||
const PENDING_IDLE_MS = 1_000;
|
||||
const READ_BLOCK_MS = 5_000;
|
||||
const READ_COUNT = 10;
|
||||
const PENDING_BATCH_SIZE = 20;
|
||||
const STOP_TIMEOUT_MS = 10_000;
|
||||
|
||||
type BackgroundJobType = "shadow-eval" | "summarize";
|
||||
|
||||
interface ShadowEvalJob {
|
||||
classified: ClassifiedIntent;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
|
||||
interface EvalObservation {
|
||||
llmJudgeErrored: boolean;
|
||||
llmOverall: number | null;
|
||||
recordedAt: number;
|
||||
responseLatencyMs: number | null;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function promiseWithTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
): Promise<T> {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise<T>((_, reject) => {
|
||||
setTimeout(() => {
|
||||
reject(new Error(`Timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
function getJobType(message: StreamEntry): string {
|
||||
return message.message["type"] ?? "unknown";
|
||||
}
|
||||
|
||||
function parseOptionalNumberField(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseJsonField<T>(
|
||||
value: string | undefined,
|
||||
fieldName: string,
|
||||
): T {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`Missing ${fieldName} field`);
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
||||
tier: derived.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition {
|
||||
return {
|
||||
id: `shadow:${messageId}`,
|
||||
lineNumber: 0,
|
||||
input: {
|
||||
request: job.request,
|
||||
},
|
||||
expected: {},
|
||||
};
|
||||
}
|
||||
|
||||
function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
const input = parseJsonField<{ request: AgentRequest }>(
|
||||
message.message["input"],
|
||||
"input",
|
||||
);
|
||||
|
||||
return {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
message.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
||||
derived: parseJsonField<DerivedState>(
|
||||
message.message["derived"],
|
||||
"derived",
|
||||
),
|
||||
latencyMs: parseOptionalNumberField(message.message["latencyMs"]),
|
||||
request: input.request,
|
||||
response: parseJsonField<AgentResponse>(
|
||||
message.message["output"],
|
||||
"output",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function calculatePercentile(
|
||||
values: number[],
|
||||
percentile: number,
|
||||
): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.min(
|
||||
sorted.length - 1,
|
||||
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
||||
);
|
||||
const selected = sorted[index];
|
||||
return selected === undefined ? null : selected;
|
||||
}
|
||||
|
||||
export function calculateEvalScoreDropPct(
|
||||
currentAverage: number | null,
|
||||
baselineAverage: number | null,
|
||||
): number | null {
|
||||
if (
|
||||
currentAverage === null ||
|
||||
baselineAverage === null ||
|
||||
baselineAverage <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return ((baselineAverage - currentAverage) / baselineAverage) * 100;
|
||||
}
|
||||
|
||||
export function calculateLlmErrorRate(
|
||||
observations: EvalObservation[],
|
||||
): number | null {
|
||||
if (observations.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const errorCount = observations.filter(
|
||||
(observation) => observation.llmJudgeErrored,
|
||||
).length;
|
||||
return errorCount / observations.length;
|
||||
}
|
||||
|
||||
export class ShadowWorker {
|
||||
private lastAggregateAt = 0;
|
||||
private loopPromise: Promise<void> | null = null;
|
||||
private readonly logger = getLogger();
|
||||
private readonly recentEvalObservations: EvalObservation[] = [];
|
||||
private stopRequested = false;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
private readonly llmClient: LlmClient,
|
||||
private readonly conversationStore: ConversationStore,
|
||||
private readonly alertSender?: AlertSender,
|
||||
private readonly consumerName = `bg-worker-${process.pid}`,
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.ensureConsumerGroup();
|
||||
|
||||
if (this.loopPromise) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.stopRequested = false;
|
||||
this.loopPromise = this.runLoop();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopRequested = true;
|
||||
|
||||
if (!this.loopPromise) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{ err: error },
|
||||
"[ShadowWorker] Stop timed out while waiting for the loop to finish",
|
||||
);
|
||||
} finally {
|
||||
this.loopPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureConsumerGroup(): Promise<void> {
|
||||
try {
|
||||
await this.garnetClient.createConsumerGroup(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!String(error).includes("BUSYGROUP")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runLoop(): Promise<void> {
|
||||
while (!this.stopRequested) {
|
||||
try {
|
||||
await this.runPeriodicChecksIfNeeded();
|
||||
|
||||
const recoveredPending = await this.recoverPendingOnce();
|
||||
if (recoveredPending) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const streams = await this.garnetClient.readGroup(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
this.consumerName,
|
||||
">",
|
||||
{
|
||||
blockMs: READ_BLOCK_MS,
|
||||
count: READ_COUNT,
|
||||
},
|
||||
);
|
||||
|
||||
if (!streams) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const stream of streams) {
|
||||
for (const message of stream.messages) {
|
||||
if (this.stopRequested) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.processMessage(
|
||||
message,
|
||||
message.deliveriesCounter ?? 1,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ err: error },
|
||||
"[ShadowWorker] Loop iteration failed",
|
||||
);
|
||||
await delay(1_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async recoverPendingOnce(): Promise<boolean> {
|
||||
const pending = await this.garnetClient.getPendingStreamRange(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
PENDING_BATCH_SIZE,
|
||||
);
|
||||
const recoverable = pending.filter(
|
||||
(entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS,
|
||||
);
|
||||
|
||||
if (recoverable.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const entry of recoverable) {
|
||||
if (this.stopRequested) {
|
||||
break;
|
||||
}
|
||||
|
||||
await this.recoverPendingEntry(entry);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async recoverPendingEntry(
|
||||
entry: StreamPendingEntry,
|
||||
): Promise<void> {
|
||||
const nextDeliveryCount = entry.deliveriesCounter + 1;
|
||||
const claimed = await this.garnetClient.claimPendingStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
this.consumerName,
|
||||
PENDING_IDLE_MS,
|
||||
entry.id,
|
||||
{
|
||||
retryCount: nextDeliveryCount,
|
||||
},
|
||||
);
|
||||
const message = claimed[0];
|
||||
|
||||
if (!message) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (nextDeliveryCount > MAX_DELIVERIES) {
|
||||
await this.handleMaxDeliveries(
|
||||
message,
|
||||
nextDeliveryCount,
|
||||
"recovered entry exceeded max deliveries",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.processMessage(message, nextDeliveryCount);
|
||||
}
|
||||
|
||||
private async processMessage(
|
||||
message: StreamEntry,
|
||||
deliveryCount: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
switch (getJobType(message) as BackgroundJobType) {
|
||||
case "shadow-eval":
|
||||
await this.processShadowEval(message);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
case "summarize":
|
||||
await this.processSummarize(message);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
default:
|
||||
this.logger.warn(
|
||||
{
|
||||
messageId: message.id,
|
||||
type: getJobType(message),
|
||||
},
|
||||
"[ShadowWorker] Unknown background job type; acknowledging",
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{
|
||||
err: error,
|
||||
deliveryCount,
|
||||
messageId: message.id,
|
||||
type: getJobType(message),
|
||||
},
|
||||
"[ShadowWorker] Job processing failed",
|
||||
);
|
||||
|
||||
if (deliveryCount >= MAX_DELIVERIES) {
|
||||
await this.handleMaxDeliveries(
|
||||
message,
|
||||
deliveryCount,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async processShadowEval(message: StreamEntry): Promise<void> {
|
||||
const job = parseShadowEvalJob(message);
|
||||
const validation = validateOnboardingResponse(
|
||||
job.response.text,
|
||||
job.derived,
|
||||
{
|
||||
allowShortResponse: job.derived.allowShortResponse,
|
||||
},
|
||||
);
|
||||
const llmJudge = await judgeOnboardingWithLlm(
|
||||
this.llmClient,
|
||||
toEvalCase(job, message.id),
|
||||
job.response.text,
|
||||
toDerivedSnapshot(job.derived),
|
||||
);
|
||||
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
||||
const llmJudgeErrored =
|
||||
llmJudge.skipped &&
|
||||
llmJudge.reason !== "empty-response" &&
|
||||
llmJudge.reason !== "disabled";
|
||||
|
||||
this.recordEvalObservation({
|
||||
llmJudgeErrored,
|
||||
llmOverall,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: job.latencyMs ?? null,
|
||||
});
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval-result",
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs:
|
||||
job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall:
|
||||
llmOverall === null ? "" : String(llmOverall),
|
||||
input: JSON.stringify({
|
||||
request: job.request,
|
||||
}),
|
||||
output: JSON.stringify(job.response),
|
||||
derived: JSON.stringify(job.derived),
|
||||
classified: JSON.stringify(job.classified),
|
||||
validation: JSON.stringify(validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
maxLen: getEvalStreamMaxLen(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async processSummarize(message: StreamEntry): Promise<void> {
|
||||
const chatroomId = message.message["chatroomId"];
|
||||
const userId = message.message["userId"];
|
||||
|
||||
if (!chatroomId?.trim() || !userId?.trim()) {
|
||||
throw new Error("Missing summarize job identifiers");
|
||||
}
|
||||
|
||||
const [history, existingSummary] = await Promise.all([
|
||||
this.conversationStore.getHistory(chatroomId, userId),
|
||||
this.conversationStore.getSummary(chatroomId, userId),
|
||||
]);
|
||||
|
||||
if (history.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const summary = await summarizeConversation(
|
||||
this.llmClient,
|
||||
history,
|
||||
existingSummary ?? undefined,
|
||||
);
|
||||
|
||||
await this.conversationStore.setSummary(chatroomId, userId, summary);
|
||||
}
|
||||
|
||||
private async handleMaxDeliveries(
|
||||
message: StreamEntry,
|
||||
deliveryCount: number,
|
||||
lastError: string,
|
||||
): Promise<void> {
|
||||
const type = getJobType(message);
|
||||
|
||||
if (type === "summarize") {
|
||||
this.logger.warn(
|
||||
{
|
||||
deliveryCount,
|
||||
lastError,
|
||||
messageId: message.id,
|
||||
},
|
||||
"[ShadowWorker] Dropping summarize job after max retries",
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
BACKGROUND_DLQ_STREAM_KEY,
|
||||
{
|
||||
type,
|
||||
payload: JSON.stringify(message.message),
|
||||
retryCount: String(deliveryCount),
|
||||
lastError: lastError.slice(0, 500),
|
||||
originalId: message.id,
|
||||
},
|
||||
{
|
||||
maxLen: 1_000,
|
||||
},
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
}
|
||||
|
||||
private acknowledge(id: string): Promise<number> {
|
||||
return this.garnetClient.acknowledgeStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
private recordEvalObservation(observation: EvalObservation): void {
|
||||
this.recentEvalObservations.push(observation);
|
||||
this.pruneOldObservations(Date.now());
|
||||
}
|
||||
|
||||
private pruneOldObservations(now: number): void {
|
||||
const cutoff = now - ALERT_WINDOW_MS;
|
||||
let removeCount = 0;
|
||||
|
||||
for (const observation of this.recentEvalObservations) {
|
||||
if (observation.recordedAt >= cutoff) {
|
||||
break;
|
||||
}
|
||||
removeCount += 1;
|
||||
}
|
||||
|
||||
if (removeCount > 0) {
|
||||
this.recentEvalObservations.splice(0, removeCount);
|
||||
}
|
||||
}
|
||||
|
||||
private async runPeriodicChecksIfNeeded(): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastAggregateAt = now;
|
||||
this.pruneOldObservations(now);
|
||||
|
||||
await this.checkLatencyAlert(now);
|
||||
await this.checkEvalScoreDropAlert(now);
|
||||
await this.checkLlmErrorRateAlert(now);
|
||||
await this.checkOpenCircuitAlerts(now);
|
||||
}
|
||||
|
||||
private async checkLatencyAlert(now: number): Promise<void> {
|
||||
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
|
||||
);
|
||||
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
||||
const threshold = getAlertThresholdP95LatencyMs();
|
||||
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: p95LatencyMs,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkEvalScoreDropAlert(now: number): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.llmOverall === null ? [] : [observation.llmOverall],
|
||||
);
|
||||
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentAverage =
|
||||
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
||||
const baselineAverage = await this.loadBaselineOverallScore();
|
||||
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
||||
const threshold = getAlertThresholdEvalScoreDropPct();
|
||||
|
||||
if (dropPct === null || dropPct <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "eval-score-drop",
|
||||
metric: "shadow_eval.llm_overall_drop_pct",
|
||||
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(dropPct.toFixed(3)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkLlmErrorRateAlert(now: number): Promise<void> {
|
||||
if (
|
||||
!this.alertSender ||
|
||||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
|
||||
const threshold = getAlertThresholdLlmErrorRate();
|
||||
if (errorRate === null || errorRate <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "llm-error-rate",
|
||||
metric: "shadow_eval.llm_error_rate",
|
||||
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(errorRate.toFixed(4)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkOpenCircuitAlerts(now: number): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*");
|
||||
|
||||
for (const key of circuitKeys) {
|
||||
const record = await this.garnetClient.getHash(key);
|
||||
if (record["state"] !== "open") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const endpointHash = key.split(":").at(-1) ?? "unknown";
|
||||
const failureCount = Number(record["failureCount"]);
|
||||
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "endpoint-down",
|
||||
metric: "llm.circuit.open",
|
||||
severity: "critical",
|
||||
suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.",
|
||||
threshold: 0,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number.isFinite(failureCount) ? failureCount : 1,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
},
|
||||
{
|
||||
dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadBaselineOverallScore(): Promise<number | null> {
|
||||
const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY);
|
||||
const redisParsed = parseBaselineOverallFromRaw(redisBaseline);
|
||||
if (redisParsed !== null) {
|
||||
return redisParsed;
|
||||
}
|
||||
|
||||
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const fileRaw = await readFile(filePath, "utf8");
|
||||
return parseBaselineOverallFromRaw(fileRaw);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{ err: error, filePath },
|
||||
"[ShadowWorker] Failed to read baseline file",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseBaselineOverallFromRaw(raw: string | null): number | null {
|
||||
if (!raw?.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as Partial<EvalBaselineFile>;
|
||||
const value = parsed.summary?.llmAverageOverall;
|
||||
return typeof value === "number" ? value : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type {
|
||||
AgentRequest,
|
||||
AgentResponse,
|
||||
ConversationMessage,
|
||||
} from "@agent/types";
|
||||
import type { Blackboard, OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { ClassifiedIntent, DerivedState } from "@skills/onboarding/state/onboarding-state";
|
||||
|
||||
export interface EvalRoomContextMessage {
|
||||
senderId: string;
|
||||
nickname: string;
|
||||
message: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface EvalCaseInput {
|
||||
request: AgentRequest;
|
||||
conversationHistory?: ConversationMessage[] | undefined;
|
||||
roomContext?: EvalRoomContextMessage[] | undefined;
|
||||
session?: OnboardingSession | null | undefined;
|
||||
blackboard?: Blackboard | null | undefined;
|
||||
summary?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalCaseExpected {
|
||||
intent?: string | undefined;
|
||||
mustIncludeAny?: string[] | undefined;
|
||||
mustNotInclude?: string[] | undefined;
|
||||
responseMode?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalCaseDefinition {
|
||||
id: string;
|
||||
lineNumber: number;
|
||||
name?: string | undefined;
|
||||
notes?: string | undefined;
|
||||
input: EvalCaseInput;
|
||||
expected: EvalCaseExpected;
|
||||
}
|
||||
|
||||
export interface RuleJudgeCheck {
|
||||
name:
|
||||
| "validator"
|
||||
| "intent"
|
||||
| "responseMode"
|
||||
| "mustIncludeAny"
|
||||
| "mustNotInclude";
|
||||
passed: boolean;
|
||||
skipped: boolean;
|
||||
detail?: string | undefined;
|
||||
}
|
||||
|
||||
export interface RuleJudgeResult {
|
||||
passed: boolean;
|
||||
score: number;
|
||||
checks: RuleJudgeCheck[];
|
||||
validatorStatus: "pass" | "soft-warn" | "reject";
|
||||
validatorReasons: string[];
|
||||
}
|
||||
|
||||
export interface LlmJudgeScores {
|
||||
friendliness: number;
|
||||
accuracy: number;
|
||||
concision: number;
|
||||
scope: number;
|
||||
overall: number;
|
||||
}
|
||||
|
||||
export type LlmJudgeResult =
|
||||
| {
|
||||
skipped: true;
|
||||
reason: string;
|
||||
raw?: string | undefined;
|
||||
}
|
||||
| {
|
||||
skipped: false;
|
||||
model: string;
|
||||
endpoint: string;
|
||||
provider: string;
|
||||
scores: LlmJudgeScores;
|
||||
rationale: string;
|
||||
raw: string;
|
||||
};
|
||||
|
||||
export interface EvalDerivedSnapshot {
|
||||
allowShortResponse: boolean;
|
||||
intent: string;
|
||||
phase: string;
|
||||
responseMode: DerivedState["responseMode"];
|
||||
shouldRespond: boolean;
|
||||
suggestedMaxTokens: number;
|
||||
tier: DerivedState["tier"];
|
||||
}
|
||||
|
||||
export interface EvalClassifiedSnapshot {
|
||||
confidence: number;
|
||||
intent: string;
|
||||
source: ClassifiedIntent["source"];
|
||||
}
|
||||
|
||||
export interface EvalCaseResult {
|
||||
id: string;
|
||||
lineNumber: number;
|
||||
name?: string | undefined;
|
||||
latencyMs: number;
|
||||
classified: EvalClassifiedSnapshot;
|
||||
derived: EvalDerivedSnapshot;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
ruleJudge: RuleJudgeResult;
|
||||
llmJudge: LlmJudgeResult;
|
||||
error?: string | undefined;
|
||||
}
|
||||
|
||||
export interface EvalReportSummary {
|
||||
averageLatencyMs: number;
|
||||
cases: number;
|
||||
emptyResponses: number;
|
||||
llmAverageAccuracy: number | null;
|
||||
llmAverageConcision: number | null;
|
||||
llmAverageFriendliness: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
llmAverageScope: number | null;
|
||||
llmJudgedCases: number;
|
||||
passRate: number;
|
||||
passedCases: number;
|
||||
ruleAverageScore: number;
|
||||
}
|
||||
|
||||
export interface EvalSummaryDelta {
|
||||
averageLatencyMs: number | null;
|
||||
llmAverageOverall: number | null;
|
||||
passRate: number | null;
|
||||
ruleAverageScore: number | null;
|
||||
}
|
||||
|
||||
export interface EvalReport {
|
||||
baselineDelta?: EvalSummaryDelta | undefined;
|
||||
generatedAt: string;
|
||||
inputPath: string;
|
||||
setName: string;
|
||||
summary: EvalReportSummary;
|
||||
cases: EvalCaseResult[];
|
||||
}
|
||||
|
||||
export interface EvalBaselineFile {
|
||||
reportPath: string;
|
||||
setName: string;
|
||||
summary: EvalReportSummary;
|
||||
updatedAt: string;
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
export type {
|
||||
ChatCompletionOptions,
|
||||
ChatCompletionResult,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatMessage,
|
||||
ChatToolCall,
|
||||
LlmClientOptions,
|
||||
LlmEndpointConfig,
|
||||
} from "./types.js";
|
||||
|
||||
+76
-71
@@ -1,49 +1,37 @@
|
||||
import type {
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionOptions,
|
||||
ChatMessage,
|
||||
ChatCompletionResult,
|
||||
ChatCompletionResponse,
|
||||
LlmClientOptions,
|
||||
LlmEndpointConfig,
|
||||
ProviderName,
|
||||
} from "./types.js";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import { withLlmSpan } from "@obs/spans";
|
||||
import {
|
||||
getLlmHealthTimeoutMs,
|
||||
getLlmMaxTokens,
|
||||
getLlmRequestTimeoutMs,
|
||||
getLlmTemperature,
|
||||
} from "../config.js";
|
||||
import { LlmCircuitBreaker } from "./circuit.js";
|
||||
import { getProviderAdapter } from "./providers/index.js";
|
||||
import { selectMaxTokensForTier, selectModelForProvider, selectProviderOrder } from "./routing.js";
|
||||
|
||||
const DEFAULT_MODEL = "Qwen3.5-4B";
|
||||
|
||||
function buildCliProxyHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, "");
|
||||
return `${normalized.replace(/\/+$/u, "")}/`;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeEndpointConfig(
|
||||
endpoint: string | { url: string; provider?: "default" | "cliproxy" },
|
||||
endpoint: string | { url: string; provider?: ProviderName | "default" | undefined },
|
||||
): LlmEndpointConfig {
|
||||
const raw =
|
||||
typeof endpoint === "string"
|
||||
? { provider: "default" as const, url: endpoint }
|
||||
: { provider: endpoint.provider ?? "default", url: endpoint.url };
|
||||
? { provider: "llama" as const, url: endpoint }
|
||||
: { provider: endpoint.provider === "default" ? "llama" : (endpoint.provider ?? "llama"), url: endpoint.url };
|
||||
const provider = getProviderAdapter(raw.provider);
|
||||
|
||||
return {
|
||||
completionUrl: buildCompletionUrl(raw.url),
|
||||
healthUrl:
|
||||
raw.provider === "cliproxy"
|
||||
? buildCliProxyHealthUrl(raw.url)
|
||||
: buildHealthUrl(raw.url),
|
||||
healthUrl: provider.createHealthUrl(raw.url),
|
||||
provider: raw.provider,
|
||||
};
|
||||
}
|
||||
@@ -65,49 +53,68 @@ function buildCompletionUrl(endpoint: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
function buildHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/health";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
if (endpoint.includes("/v1/chat/completions")) {
|
||||
return `${endpoint.split("/v1/chat/completions")[0]}/health`;
|
||||
}
|
||||
return `${endpoint.replace(/\/+$/u, "")}/health`;
|
||||
function getRequestSignal(
|
||||
timeoutMs: number,
|
||||
externalSignal?: AbortSignal,
|
||||
): AbortSignal {
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
|
||||
if (!externalSignal) {
|
||||
return timeoutSignal;
|
||||
}
|
||||
|
||||
if (typeof AbortSignal.any === "function") {
|
||||
return AbortSignal.any([timeoutSignal, externalSignal]);
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const abort = (): void => controller.abort();
|
||||
|
||||
timeoutSignal.addEventListener("abort", abort, { once: true });
|
||||
externalSignal.addEventListener("abort", abort, { once: true });
|
||||
|
||||
return controller.signal;
|
||||
}
|
||||
|
||||
export class LlmClient {
|
||||
private readonly circuit: LlmCircuitBreaker;
|
||||
private readonly endpoints: LlmEndpointConfig[];
|
||||
private readonly apiKey: string | undefined;
|
||||
private readonly model: string;
|
||||
private readonly llamaModel: string;
|
||||
private readonly providerApiKeys: Partial<Record<ProviderName, string>>;
|
||||
|
||||
constructor(options: LlmClientOptions) {
|
||||
this.circuit = new LlmCircuitBreaker(options.garnetClient);
|
||||
this.endpoints = options.endpoints.map(normalizeEndpointConfig);
|
||||
this.apiKey = options.apiKey;
|
||||
this.model = options.model ?? DEFAULT_MODEL;
|
||||
this.llamaModel = options.model ?? DEFAULT_MODEL;
|
||||
this.providerApiKeys = options.providerApiKeys ?? {};
|
||||
}
|
||||
|
||||
private getHeaders(includeJson = false): Record<string, string> {
|
||||
private getHeaders(
|
||||
provider: ProviderName,
|
||||
includeJson = false,
|
||||
): Record<string, string> {
|
||||
const headers: Record<string, string> = {};
|
||||
if (includeJson) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
if (this.apiKey) {
|
||||
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
||||
const apiKey = this.providerApiKeys[provider] ?? this.apiKey;
|
||||
if (apiKey) {
|
||||
headers["Authorization"] = `Bearer ${apiKey}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
private async checkHealth(endpoint: LlmEndpointConfig): Promise<boolean> {
|
||||
const provider = getProviderAdapter(endpoint.provider);
|
||||
if (!provider.supportsHealthCheck) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(endpoint.healthUrl, {
|
||||
headers: this.getHeaders(),
|
||||
headers: this.getHeaders(endpoint.provider),
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(getLlmHealthTimeoutMs()),
|
||||
});
|
||||
@@ -122,12 +129,8 @@ export class LlmClient {
|
||||
|
||||
async chatCompletion(
|
||||
messages: ChatMessage[],
|
||||
options?: {
|
||||
jsonMode?: boolean | undefined;
|
||||
maxTokens?: number | undefined;
|
||||
temperature?: number | undefined;
|
||||
},
|
||||
): Promise<{ content: string; endpoint: string } | null> {
|
||||
options?: ChatCompletionOptions,
|
||||
): Promise<ChatCompletionResult | null> {
|
||||
const logger = getRequestLogger();
|
||||
|
||||
if (this.endpoints.length === 0) {
|
||||
@@ -135,18 +138,10 @@ export class LlmClient {
|
||||
return null;
|
||||
}
|
||||
|
||||
const request: ChatCompletionRequest = {
|
||||
model: this.model,
|
||||
messages,
|
||||
max_tokens: options?.maxTokens ?? getLlmMaxTokens(),
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
};
|
||||
const orderedEndpoints = selectProviderOrder(this.endpoints, options);
|
||||
|
||||
if (options?.jsonMode) {
|
||||
request.response_format = { type: "json_object" };
|
||||
}
|
||||
|
||||
for (const endpoint of this.endpoints) {
|
||||
for (const endpoint of orderedEndpoints) {
|
||||
const provider = getProviderAdapter(endpoint.provider);
|
||||
const healthy = await this.circuit.canUse(
|
||||
endpoint.completionUrl,
|
||||
() => this.checkHealth(endpoint),
|
||||
@@ -155,7 +150,7 @@ export class LlmClient {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
},
|
||||
"[LLM] Endpoint unavailable via circuit breaker",
|
||||
);
|
||||
@@ -163,17 +158,30 @@ export class LlmClient {
|
||||
}
|
||||
|
||||
try {
|
||||
const requestTimeoutMs = Math.max(
|
||||
1,
|
||||
Math.min(getLlmRequestTimeoutMs(), options?.deadlineMs ?? Number.POSITIVE_INFINITY),
|
||||
);
|
||||
const request = provider.prepareRequest({
|
||||
jsonMode: options?.jsonMode,
|
||||
maxTokens: selectMaxTokensForTier(options),
|
||||
messages,
|
||||
model: selectModelForProvider(provider.name, options, this.llamaModel),
|
||||
temperature: options?.temperature ?? getLlmTemperature(),
|
||||
tools: options?.tools,
|
||||
});
|
||||
|
||||
const llmResult = await withLlmSpan(
|
||||
{
|
||||
"gen_ai.request.model": request.model,
|
||||
"gen_ai.system": endpoint.provider ?? 'default',
|
||||
"gen_ai.system": provider.name,
|
||||
},
|
||||
async (span) => {
|
||||
const response = await fetch(endpoint.completionUrl, {
|
||||
body: JSON.stringify(request),
|
||||
headers: this.getHeaders(true),
|
||||
headers: this.getHeaders(provider.name, true),
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(getLlmRequestTimeoutMs()),
|
||||
signal: getRequestSignal(requestTimeoutMs, options?.signal),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -181,7 +189,7 @@ export class LlmClient {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
status: response.status,
|
||||
body,
|
||||
},
|
||||
@@ -191,7 +199,7 @@ export class LlmClient {
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as ChatCompletionResponse;
|
||||
const content = payload.choices?.[0]?.message?.content;
|
||||
const result = provider.toResult(payload, endpoint.completionUrl);
|
||||
|
||||
span.setAttributes({
|
||||
"gen_ai.response.finish_reasons": payload.choices
|
||||
@@ -201,21 +209,18 @@ export class LlmClient {
|
||||
"gen_ai.usage.output_tokens": payload.usage?.completion_tokens,
|
||||
});
|
||||
|
||||
if (typeof content !== "string" || !content.trim()) {
|
||||
if (!result) {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
},
|
||||
"[LLM] Empty response from endpoint",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
content: content.trim(),
|
||||
endpoint: endpoint.completionUrl,
|
||||
};
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
||||
@@ -231,7 +236,7 @@ export class LlmClient {
|
||||
logger.warn(
|
||||
{
|
||||
endpoint: endpoint.completionUrl,
|
||||
provider: endpoint.provider ?? 'default',
|
||||
provider: provider.name,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
"[LLM] Request error",
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { buildOpenAiRequest, normalizeProviderResponse } from "./shared.js";
|
||||
import type { ProviderAdapter } from "./types.js";
|
||||
|
||||
export const cliproxyProvider: ProviderAdapter = {
|
||||
name: "cliproxy",
|
||||
capabilities: {
|
||||
jsonMode: true,
|
||||
promptCache: false,
|
||||
toolCalls: "openai",
|
||||
},
|
||||
createHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
const normalized = endpoint.replace(/\/v1\/chat\/completions\/?$/u, "");
|
||||
return `${normalized.replace(/\/+$/u, "")}/`;
|
||||
}
|
||||
},
|
||||
prepareRequest(input) {
|
||||
return buildOpenAiRequest(input);
|
||||
},
|
||||
supportsHealthCheck: true,
|
||||
toResult(payload, endpoint) {
|
||||
return normalizeProviderResponse(payload, endpoint, "cliproxy");
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { ProviderName } from "../types.js";
|
||||
import { cliproxyProvider } from "./cliproxy.js";
|
||||
import { llamaProvider } from "./llama.js";
|
||||
import { openrouterProvider } from "./openrouter.js";
|
||||
import type { ProviderAdapter } from "./types.js";
|
||||
|
||||
const PROVIDERS: Record<ProviderName, ProviderAdapter> = {
|
||||
cliproxy: cliproxyProvider,
|
||||
llama: llamaProvider,
|
||||
openrouter: openrouterProvider,
|
||||
};
|
||||
|
||||
export function getProviderAdapter(name: ProviderName): ProviderAdapter {
|
||||
return PROVIDERS[name];
|
||||
}
|
||||
|
||||
export { cliproxyProvider, llamaProvider, openrouterProvider };
|
||||
export type { ProviderAdapter, ProviderRequestInput } from "./types.js";
|
||||
@@ -0,0 +1,32 @@
|
||||
import { buildHermesRequest, normalizeProviderResponse } from "./shared.js";
|
||||
import type { ProviderAdapter } from "./types.js";
|
||||
|
||||
export const llamaProvider: ProviderAdapter = {
|
||||
name: "llama",
|
||||
capabilities: {
|
||||
jsonMode: false,
|
||||
promptCache: false,
|
||||
toolCalls: "hermes",
|
||||
},
|
||||
createHealthUrl(endpoint: string): string {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
url.pathname = "/health";
|
||||
url.search = "";
|
||||
url.hash = "";
|
||||
return url.toString();
|
||||
} catch {
|
||||
if (endpoint.includes("/v1/chat/completions")) {
|
||||
return `${endpoint.split("/v1/chat/completions")[0]}/health`;
|
||||
}
|
||||
return `${endpoint.replace(/\/+$/u, "")}/health`;
|
||||
}
|
||||
},
|
||||
prepareRequest(input) {
|
||||
return buildHermesRequest(input);
|
||||
},
|
||||
supportsHealthCheck: true,
|
||||
toResult(payload, endpoint) {
|
||||
return normalizeProviderResponse(payload, endpoint, "llama");
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
import { buildOpenAiRequest, normalizeProviderResponse } from "./shared.js";
|
||||
import type { ProviderAdapter } from "./types.js";
|
||||
|
||||
export const openrouterProvider: ProviderAdapter = {
|
||||
name: "openrouter",
|
||||
capabilities: {
|
||||
jsonMode: true,
|
||||
promptCache: false,
|
||||
toolCalls: "openai",
|
||||
},
|
||||
createHealthUrl(endpoint: string): string {
|
||||
return endpoint;
|
||||
},
|
||||
prepareRequest(input) {
|
||||
return buildOpenAiRequest(input);
|
||||
},
|
||||
supportsHealthCheck: false,
|
||||
toResult(payload, endpoint) {
|
||||
return normalizeProviderResponse(payload, endpoint, "openrouter");
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,235 @@
|
||||
import { serializeToolsForHermes, serializeToolsForOpenAi } from "@tools/serializer";
|
||||
import type { Tool } from "@tools/tool";
|
||||
import type {
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResult,
|
||||
ChatMessage,
|
||||
ChatToolCall,
|
||||
} from "../types.js";
|
||||
import type { ProviderRequestInput } from "./types.js";
|
||||
|
||||
function parseJsonSafely(value: string): unknown {
|
||||
try {
|
||||
return JSON.parse(value) as unknown;
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function serializeHermesToolCalls(toolCalls: readonly ChatToolCall[]): string {
|
||||
const payload = toolCalls.map((toolCall) => ({
|
||||
id: toolCall.id,
|
||||
name: toolCall.function.name,
|
||||
arguments: parseJsonSafely(toolCall.function.arguments),
|
||||
}));
|
||||
|
||||
return `<tool_call>${JSON.stringify(payload)}</tool_call>`;
|
||||
}
|
||||
|
||||
function serializeHermesToolResult(message: ChatMessage): string {
|
||||
return `<tool_result>${JSON.stringify({
|
||||
name: message.name ?? message.tool_call_id ?? "tool",
|
||||
result: parseJsonSafely(message.content),
|
||||
})}</tool_result>`;
|
||||
}
|
||||
|
||||
function injectHermesToolBlock(
|
||||
messages: ChatMessage[],
|
||||
tools: readonly Tool<unknown, unknown>[] | undefined,
|
||||
): ChatMessage[] {
|
||||
if (!tools || tools.length === 0) {
|
||||
return [...messages];
|
||||
}
|
||||
|
||||
const toolBlock = serializeToolsForHermes(tools);
|
||||
const cloned = messages.map((message) => ({ ...message }));
|
||||
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
|
||||
|
||||
if (firstSystemIndex === -1) {
|
||||
cloned.unshift({
|
||||
role: "system",
|
||||
content: toolBlock,
|
||||
});
|
||||
return cloned;
|
||||
}
|
||||
|
||||
const current = cloned[firstSystemIndex];
|
||||
if (!current) {
|
||||
return cloned;
|
||||
}
|
||||
|
||||
if (current.content.includes("<tools>")) {
|
||||
return cloned;
|
||||
}
|
||||
|
||||
cloned[firstSystemIndex] = {
|
||||
...current,
|
||||
content: `${current.content}\n\n${toolBlock}`,
|
||||
};
|
||||
return cloned;
|
||||
}
|
||||
|
||||
function injectJsonModeReminder(messages: ChatMessage[]): ChatMessage[] {
|
||||
const cloned = messages.map((message) => ({ ...message }));
|
||||
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
|
||||
const reminder = "반드시 JSON object 하나만 출력하세요. 설명 문장은 쓰지 마세요.";
|
||||
|
||||
if (firstSystemIndex === -1) {
|
||||
cloned.unshift({
|
||||
role: "system",
|
||||
content: reminder,
|
||||
});
|
||||
return cloned;
|
||||
}
|
||||
|
||||
const current = cloned[firstSystemIndex];
|
||||
if (!current || current.content.includes(reminder)) {
|
||||
return cloned;
|
||||
}
|
||||
|
||||
cloned[firstSystemIndex] = {
|
||||
...current,
|
||||
content: `${current.content}\n\n${reminder}`,
|
||||
};
|
||||
return cloned;
|
||||
}
|
||||
|
||||
export function normalizeOpenAiMessages(messages: ChatMessage[]): ChatMessage[] {
|
||||
return messages.map((message) => {
|
||||
if (message.role === "tool") {
|
||||
return {
|
||||
role: "tool",
|
||||
content: message.content,
|
||||
tool_call_id: message.tool_call_id,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
tool_calls: message.tool_calls,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeHermesMessages(messages: ChatMessage[]): ChatMessage[] {
|
||||
return messages.map((message) => {
|
||||
if (message.role === "tool") {
|
||||
return {
|
||||
role: "user",
|
||||
content: serializeHermesToolResult(message),
|
||||
};
|
||||
}
|
||||
|
||||
if (message.tool_calls && message.tool_calls.length > 0) {
|
||||
const parts = [
|
||||
message.content.trim(),
|
||||
serializeHermesToolCalls(message.tool_calls),
|
||||
].filter(Boolean);
|
||||
return {
|
||||
role: message.role,
|
||||
content: parts.join("\n\n"),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildOpenAiRequest(
|
||||
input: ProviderRequestInput,
|
||||
): ChatCompletionRequest {
|
||||
const request: ChatCompletionRequest = {
|
||||
model: input.model,
|
||||
messages: normalizeOpenAiMessages(input.messages),
|
||||
max_tokens: input.maxTokens,
|
||||
temperature: input.temperature,
|
||||
};
|
||||
|
||||
if (input.jsonMode) {
|
||||
request.response_format = { type: "json_object" };
|
||||
}
|
||||
|
||||
if (input.tools && input.tools.length > 0) {
|
||||
request.tools = serializeToolsForOpenAi(input.tools);
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
export function buildHermesRequest(
|
||||
input: ProviderRequestInput,
|
||||
): ChatCompletionRequest {
|
||||
let messages = injectHermesToolBlock(input.messages, input.tools);
|
||||
messages = normalizeHermesMessages(messages);
|
||||
|
||||
if (input.jsonMode) {
|
||||
messages = injectJsonModeReminder(messages);
|
||||
}
|
||||
|
||||
return {
|
||||
model: input.model,
|
||||
messages,
|
||||
max_tokens: input.maxTokens,
|
||||
temperature: input.temperature,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeToolCalls(
|
||||
toolCalls: ChatToolCall[] | null | undefined,
|
||||
): ChatToolCall[] | undefined {
|
||||
if (!toolCalls || toolCalls.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const normalized = toolCalls.flatMap((toolCall) => {
|
||||
const name = toolCall.function?.name?.trim();
|
||||
const args = toolCall.function?.arguments;
|
||||
if (!name || typeof args !== "string") {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: toolCall.id,
|
||||
type: "function" as const,
|
||||
function: {
|
||||
name,
|
||||
arguments: args,
|
||||
},
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
export function normalizeProviderResponse(
|
||||
payload: ChatCompletionResponse,
|
||||
endpoint: string,
|
||||
provider: ChatCompletionResult["provider"],
|
||||
): ChatCompletionResult | null {
|
||||
const firstMessage = payload.choices?.[0]?.message;
|
||||
const content = firstMessage?.content?.trim() ?? "";
|
||||
const toolCalls = normalizeToolCalls(firstMessage?.tool_calls);
|
||||
|
||||
if (!content && (!toolCalls || toolCalls.length === 0)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
endpoint,
|
||||
message: {
|
||||
role: firstMessage?.role ?? "assistant",
|
||||
content,
|
||||
tool_calls: toolCalls,
|
||||
},
|
||||
provider,
|
||||
raw: payload,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { Tool } from "@tools/tool";
|
||||
import type {
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResult,
|
||||
ChatMessage,
|
||||
ProviderName,
|
||||
} from "../types.js";
|
||||
|
||||
export interface ProviderRequestInput {
|
||||
jsonMode?: boolean | undefined;
|
||||
maxTokens: number;
|
||||
messages: ChatMessage[];
|
||||
model: string;
|
||||
temperature: number;
|
||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||
}
|
||||
|
||||
export interface ProviderAdapter {
|
||||
capabilities: {
|
||||
jsonMode: boolean;
|
||||
promptCache: false;
|
||||
toolCalls: "openai" | "hermes" | "none";
|
||||
};
|
||||
createHealthUrl(endpoint: string): string;
|
||||
name: ProviderName;
|
||||
prepareRequest(input: ProviderRequestInput): ChatCompletionRequest;
|
||||
supportsHealthCheck: boolean;
|
||||
toResult(
|
||||
payload: ChatCompletionResponse,
|
||||
endpoint: string,
|
||||
): ChatCompletionResult | null;
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { LlmEndpointConfig } from "./types.js";
|
||||
import {
|
||||
selectMaxTokensForTier,
|
||||
selectModelForProvider,
|
||||
selectProviderOrder,
|
||||
} from "./routing.js";
|
||||
|
||||
describe("llm routing", () => {
|
||||
afterEach(() => {
|
||||
delete process.env["OPENROUTER_ENABLED"];
|
||||
delete process.env["OPENROUTER_TIER1_MODEL"];
|
||||
delete process.env["OPENROUTER_TIER2_MODEL"];
|
||||
delete process.env["TIER0_MODEL"];
|
||||
delete process.env["TIER1_PLANNER_MODEL"];
|
||||
delete process.env["TIER1_RESPONDER_MODEL"];
|
||||
delete process.env["TIER2_CRITIC_MODEL"];
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("prefers cliproxy then llama for tier 0 even when openrouter is enabled", () => {
|
||||
process.env["OPENROUTER_ENABLED"] = "true";
|
||||
const endpoints: LlmEndpointConfig[] = [
|
||||
{
|
||||
completionUrl: "http://llama.test/v1/chat/completions",
|
||||
healthUrl: "http://llama.test/health",
|
||||
provider: "llama",
|
||||
},
|
||||
{
|
||||
completionUrl: "http://openrouter.test/v1/chat/completions",
|
||||
healthUrl: "http://openrouter.test",
|
||||
provider: "openrouter",
|
||||
},
|
||||
{
|
||||
completionUrl: "http://cliproxy.test/v1/chat/completions",
|
||||
healthUrl: "http://cliproxy.test",
|
||||
provider: "cliproxy",
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
selectProviderOrder(endpoints, { tier: 0 }).map((endpoint) => endpoint.provider),
|
||||
).toEqual(["cliproxy", "llama"]);
|
||||
});
|
||||
|
||||
it("includes openrouter between cliproxy and llama for tier 1+", () => {
|
||||
process.env["OPENROUTER_ENABLED"] = "true";
|
||||
const endpoints: LlmEndpointConfig[] = [
|
||||
{
|
||||
completionUrl: "http://llama.test/v1/chat/completions",
|
||||
healthUrl: "http://llama.test/health",
|
||||
provider: "llama",
|
||||
},
|
||||
{
|
||||
completionUrl: "http://openrouter.test/v1/chat/completions",
|
||||
healthUrl: "http://openrouter.test",
|
||||
provider: "openrouter",
|
||||
},
|
||||
{
|
||||
completionUrl: "http://cliproxy.test/v1/chat/completions",
|
||||
healthUrl: "http://cliproxy.test",
|
||||
provider: "cliproxy",
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
selectProviderOrder(endpoints, { tier: 1 }).map((endpoint) => endpoint.provider),
|
||||
).toEqual(["cliproxy", "openrouter", "llama"]);
|
||||
});
|
||||
|
||||
it("selects role-aware models per provider", () => {
|
||||
process.env["TIER1_PLANNER_MODEL"] = "cliproxy-planner";
|
||||
process.env["TIER2_CRITIC_MODEL"] = "cliproxy-critic";
|
||||
process.env["OPENROUTER_TIER1_MODEL"] = "openrouter-tier1";
|
||||
process.env["OPENROUTER_TIER2_MODEL"] = "openrouter-tier2";
|
||||
|
||||
expect(
|
||||
selectModelForProvider("cliproxy", { role: "planner", tier: 1 }, "local-qwen"),
|
||||
).toBe("cliproxy-planner");
|
||||
expect(
|
||||
selectModelForProvider("cliproxy", { role: "critic", tier: 2 }, "local-qwen"),
|
||||
).toBe("cliproxy-critic");
|
||||
expect(selectModelForProvider("openrouter", { tier: 1 }, "local-qwen")).toBe(
|
||||
"openrouter-tier1",
|
||||
);
|
||||
expect(
|
||||
selectModelForProvider("openrouter", { role: "critic", tier: 2 }, "local-qwen"),
|
||||
).toBe("openrouter-tier2");
|
||||
expect(selectModelForProvider("llama", { tier: 2 }, "local-qwen")).toBe("local-qwen");
|
||||
});
|
||||
|
||||
it("chooses tier defaults for max tokens when explicit maxTokens is absent", () => {
|
||||
process.env["TIER0_MAX_TOKENS"] = "111";
|
||||
process.env["TIER1_MAX_TOKENS"] = "222";
|
||||
process.env["TIER2_MAX_TOKENS"] = "333";
|
||||
|
||||
expect(selectMaxTokensForTier({ tier: 0 })).toBe(111);
|
||||
expect(selectMaxTokensForTier({ tier: 1 })).toBe(222);
|
||||
expect(selectMaxTokensForTier({ tier: 2 })).toBe(333);
|
||||
expect(selectMaxTokensForTier({ tier: 1, maxTokens: 77 })).toBe(77);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import {
|
||||
getOpenRouterEnabled,
|
||||
getOpenRouterTier1Model,
|
||||
getOpenRouterTier2Model,
|
||||
getTier0MaxTokens,
|
||||
getTier0Model,
|
||||
getTier1MaxTokens,
|
||||
getTier1PlannerModel,
|
||||
getTier1ResponderModel,
|
||||
getTier2CriticModel,
|
||||
getTier2MaxTokens,
|
||||
getLlmMaxTokens,
|
||||
} from "../config.js";
|
||||
import type { ChatCompletionOptions, LlmEndpointConfig, ProviderName } from "./types.js";
|
||||
|
||||
export function selectModelForProvider(
|
||||
provider: ProviderName,
|
||||
options: ChatCompletionOptions | undefined,
|
||||
llamaFallbackModel: string,
|
||||
): string {
|
||||
if (options?.modelOverride?.trim()) {
|
||||
return options.modelOverride.trim();
|
||||
}
|
||||
|
||||
if (provider === "llama") {
|
||||
return llamaFallbackModel;
|
||||
}
|
||||
|
||||
if (provider === "openrouter") {
|
||||
return options?.role === "critic"
|
||||
? getOpenRouterTier2Model()
|
||||
: getOpenRouterTier1Model();
|
||||
}
|
||||
|
||||
if (options?.role === "planner") {
|
||||
return getTier1PlannerModel();
|
||||
}
|
||||
|
||||
if (options?.role === "critic") {
|
||||
return getTier2CriticModel();
|
||||
}
|
||||
|
||||
if (options?.tier === 1 || options?.tier === 2) {
|
||||
return getTier1ResponderModel();
|
||||
}
|
||||
|
||||
return getTier0Model();
|
||||
}
|
||||
|
||||
export function selectMaxTokensForTier(
|
||||
options: ChatCompletionOptions | undefined,
|
||||
): number {
|
||||
if (options?.maxTokens !== undefined) {
|
||||
return options.maxTokens;
|
||||
}
|
||||
|
||||
if (options?.tier === 0) {
|
||||
return getTier0MaxTokens();
|
||||
}
|
||||
|
||||
if (options?.tier === 1) {
|
||||
return getTier1MaxTokens();
|
||||
}
|
||||
|
||||
if (options?.tier === 2) {
|
||||
return getTier2MaxTokens();
|
||||
}
|
||||
|
||||
return getLlmMaxTokens();
|
||||
}
|
||||
|
||||
export function selectProviderOrder(
|
||||
endpoints: readonly LlmEndpointConfig[],
|
||||
options: ChatCompletionOptions | undefined,
|
||||
): LlmEndpointConfig[] {
|
||||
const preferredProviders: ProviderName[] = ["cliproxy"];
|
||||
|
||||
if (getOpenRouterEnabled() && options?.tier !== 0) {
|
||||
preferredProviders.push("openrouter");
|
||||
}
|
||||
|
||||
preferredProviders.push("llama");
|
||||
|
||||
const weighted = endpoints.map((endpoint, index) => ({
|
||||
endpoint,
|
||||
index,
|
||||
priority: preferredProviders.indexOf(endpoint.provider),
|
||||
}));
|
||||
|
||||
return weighted
|
||||
.filter((item) => item.priority !== -1)
|
||||
.sort((left, right) => {
|
||||
if (left.priority !== right.priority) {
|
||||
return left.priority - right.priority;
|
||||
}
|
||||
return left.index - right.index;
|
||||
})
|
||||
.map((item) => item.endpoint);
|
||||
}
|
||||
+44
-3
@@ -1,8 +1,23 @@
|
||||
import type { GarnetClient } from '@common/garnet';
|
||||
import type { Tool } from '@tools/tool';
|
||||
|
||||
export type ProviderName = "cliproxy" | "llama" | "openrouter";
|
||||
|
||||
export interface ChatToolCall {
|
||||
id?: string | undefined;
|
||||
type: "function";
|
||||
function: {
|
||||
name: string;
|
||||
arguments: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ChatMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
role: "system" | "user" | "assistant" | "tool";
|
||||
content: string;
|
||||
name?: string | undefined;
|
||||
tool_call_id?: string | undefined;
|
||||
tool_calls?: ChatToolCall[] | undefined;
|
||||
}
|
||||
|
||||
export interface ChatCompletionRequest {
|
||||
@@ -11,6 +26,7 @@ export interface ChatCompletionRequest {
|
||||
max_tokens: number;
|
||||
temperature: number;
|
||||
response_format?: { type: string } | undefined;
|
||||
tools?: unknown[] | undefined;
|
||||
}
|
||||
|
||||
export interface ChatCompletionResponse {
|
||||
@@ -18,7 +34,9 @@ export interface ChatCompletionResponse {
|
||||
choices?: Array<{
|
||||
finish_reason?: string | null;
|
||||
message?: {
|
||||
role?: "assistant" | "tool" | "user" | "system" | null;
|
||||
content?: string | null;
|
||||
tool_calls?: ChatToolCall[] | null;
|
||||
};
|
||||
}>;
|
||||
usage?: {
|
||||
@@ -28,15 +46,38 @@ export interface ChatCompletionResponse {
|
||||
};
|
||||
}
|
||||
|
||||
export interface ChatCompletionResult {
|
||||
content: string;
|
||||
endpoint: string;
|
||||
message: ChatMessage;
|
||||
provider: ProviderName;
|
||||
raw: ChatCompletionResponse;
|
||||
}
|
||||
|
||||
export interface ChatCompletionOptions {
|
||||
deadlineMs?: number | undefined;
|
||||
jsonMode?: boolean | undefined;
|
||||
maxTokens?: number | undefined;
|
||||
modelOverride?: string | undefined;
|
||||
role?: "planner" | "responder" | "critic" | undefined;
|
||||
signal?: AbortSignal | undefined;
|
||||
temperature?: number | undefined;
|
||||
tier?: 0 | 1 | 2 | undefined;
|
||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||
}
|
||||
|
||||
export interface LlmEndpointConfig {
|
||||
completionUrl: string;
|
||||
healthUrl: string;
|
||||
provider?: "default" | "cliproxy" | undefined;
|
||||
provider: ProviderName;
|
||||
}
|
||||
|
||||
export interface LlmClientOptions {
|
||||
endpoints: Array<string | { url: string; provider?: "default" | "cliproxy" }>;
|
||||
endpoints: Array<
|
||||
string | { url: string; provider?: ProviderName | "default" | undefined }
|
||||
>;
|
||||
garnetClient?: GarnetClient | undefined;
|
||||
apiKey?: string | undefined;
|
||||
model?: string | undefined;
|
||||
providerApiKeys?: Partial<Record<ProviderName, string>> | undefined;
|
||||
}
|
||||
+33
-11
@@ -1,5 +1,5 @@
|
||||
import { loadEnvironmentFromNodeEnv } from "./common/env-loader.js";
|
||||
import { getServiceName } from "./config.js";
|
||||
import { getOpenRouterApiUrl, getOpenRouterEnabled, getServiceName } from "./config.js";
|
||||
|
||||
import type { GarnetClient } from "./common/garnet.js";
|
||||
import type { NatsClient } from "./common/nats.js";
|
||||
@@ -21,6 +21,7 @@ const { NatsClient: RuntimeNatsClient } = await import("./common/nats.js");
|
||||
const { Agent } = await import("./agent/index.js");
|
||||
const { ConversationStore } = await import("./conversation/index.js");
|
||||
const { OnboardingSkill } = await import("./skills/onboarding/index.js");
|
||||
const { ShadowEvalBus } = await import("./eval/shadow-bus.js");
|
||||
const { LlmClient } = await import("./llm/index.js");
|
||||
const { getRawSecret, getSecret } = await import("./common/secret-manager.js");
|
||||
const { initMongoDB, closeMongoDB } = await import("./common/mongodb.js");
|
||||
@@ -53,22 +54,22 @@ function parseEndpointList(
|
||||
|
||||
function toProviderEndpoints(
|
||||
urls: string[],
|
||||
provider?: "default" | "cliproxy",
|
||||
): Array<string | { url: string; provider?: "default" | "cliproxy" }> {
|
||||
if (!provider || provider === "default") {
|
||||
provider?: "llama" | "cliproxy" | "openrouter",
|
||||
): Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }> {
|
||||
if (!provider) {
|
||||
return urls;
|
||||
}
|
||||
return urls.map((url) => ({ url, provider }));
|
||||
}
|
||||
|
||||
function formatEndpointList(
|
||||
endpoints: Array<string | { url: string; provider?: "default" | "cliproxy" }>,
|
||||
endpoints: Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }>,
|
||||
): string {
|
||||
return endpoints
|
||||
.map((endpoint) =>
|
||||
typeof endpoint === "string"
|
||||
? endpoint
|
||||
: `${endpoint.provider ?? "default"}:${endpoint.url}`,
|
||||
: `${endpoint.provider}:${endpoint.url}`,
|
||||
)
|
||||
.join(", ");
|
||||
}
|
||||
@@ -135,30 +136,50 @@ async function bootstrap(): Promise<void> {
|
||||
getRawSecret("LLAMA_SERVER_API_URL"),
|
||||
process.env["LLAMA_SERVER_API_URL"],
|
||||
);
|
||||
const openRouterEndpoints = getOpenRouterEnabled()
|
||||
? parseEndpointList(
|
||||
getRawSecret("OPENROUTER_API_URL"),
|
||||
process.env["OPENROUTER_API_URL"],
|
||||
getOpenRouterApiUrl(),
|
||||
)
|
||||
: [];
|
||||
const llmEndpoints = [
|
||||
...toProviderEndpoints(cliProxyEndpoints, "cliproxy"),
|
||||
...toProviderEndpoints(generalLlmEndpoints),
|
||||
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
|
||||
...toProviderEndpoints(generalLlmEndpoints, "llama"),
|
||||
];
|
||||
const llmApiKey = firstNonEmpty(
|
||||
const cliproxyApiKey = firstNonEmpty(
|
||||
getRawSecret("CLI_PROXY_API_KEY"),
|
||||
process.env["CLI_PROXY_API_KEY"],
|
||||
);
|
||||
const openRouterApiKey = firstNonEmpty(
|
||||
getRawSecret("OPENROUTER_API_KEY"),
|
||||
process.env["OPENROUTER_API_KEY"],
|
||||
);
|
||||
const llmApiKey = firstNonEmpty(
|
||||
getRawSecret("AGENT_LLM_API_KEY"),
|
||||
process.env["AGENT_LLM_API_KEY"],
|
||||
getRawSecret("LLAMA_SERVER_API_KEY"),
|
||||
process.env["LLAMA_SERVER_API_KEY"],
|
||||
);
|
||||
const llmModel = firstNonEmpty(
|
||||
getRawSecret("CLI_PROXY_LLM_MODEL"),
|
||||
process.env["CLI_PROXY_LLM_MODEL"],
|
||||
getRawSecret("AGENT_LLM_MODEL"),
|
||||
process.env["AGENT_LLM_MODEL"],
|
||||
);
|
||||
const providerApiKeys = Object.fromEntries(
|
||||
[
|
||||
["cliproxy", cliproxyApiKey],
|
||||
["llama", llmApiKey],
|
||||
["openrouter", openRouterApiKey],
|
||||
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
||||
) as Partial<Record<"cliproxy" | "llama" | "openrouter", string>>;
|
||||
|
||||
const llmClient = new LlmClient({
|
||||
endpoints: llmEndpoints,
|
||||
garnetClient,
|
||||
apiKey: llmApiKey,
|
||||
model: llmModel,
|
||||
providerApiKeys,
|
||||
});
|
||||
|
||||
if (llmEndpoints.length > 0) {
|
||||
@@ -197,7 +218,8 @@ async function bootstrap(): Promise<void> {
|
||||
const agent = new Agent(natsClient, conversationStore);
|
||||
|
||||
// Register skills
|
||||
const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService);
|
||||
const shadowEvalBus = new ShadowEvalBus(garnetClient);
|
||||
const onboardingSkill = new OnboardingSkill(garnetClient, llmClient, roomContextService, shadowEvalBus);
|
||||
agent.registerSkill(onboardingSkill);
|
||||
|
||||
installShutdownHooks(async () => {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import { AGENT_ALERT_SUBJECT } from "@common/nats-subjects";
|
||||
import { AlertService } from "./alert.js";
|
||||
|
||||
const payload: AlertPayload = {
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: "warn",
|
||||
threshold: 3000,
|
||||
timestamp: "2026-04-17T00:00:00.000Z",
|
||||
value: 4100,
|
||||
window: "5m",
|
||||
};
|
||||
|
||||
describe("AlertService", () => {
|
||||
it("publishes when dedupe key is claimed", async () => {
|
||||
const setStringIfAbsent = vi.fn().mockResolvedValue(true);
|
||||
const publish = vi.fn().mockResolvedValue(undefined);
|
||||
const service = new AlertService({ setStringIfAbsent }, { publish });
|
||||
|
||||
const result = await service.sendAlert(payload);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(setStringIfAbsent).toHaveBeenCalledTimes(1);
|
||||
expect(publish).toHaveBeenCalledWith(AGENT_ALERT_SUBJECT, payload);
|
||||
});
|
||||
|
||||
it("suppresses duplicate alerts", async () => {
|
||||
const setStringIfAbsent = vi.fn().mockResolvedValue(false);
|
||||
const publish = vi.fn().mockResolvedValue(undefined);
|
||||
const service = new AlertService({ setStringIfAbsent }, { publish });
|
||||
|
||||
const result = await service.sendAlert(payload);
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(publish).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { AGENT_ALERT_SUBJECT, type AlertPayload } from "@common/nats-subjects";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import { getAlertRateLimitSec } from "../config.js";
|
||||
import { getLogger } from "./logger.js";
|
||||
|
||||
export interface AlertSender {
|
||||
sendAlert(
|
||||
payload: AlertPayload,
|
||||
options?: {
|
||||
dedupeKey?: string | undefined;
|
||||
},
|
||||
): Promise<boolean>;
|
||||
}
|
||||
|
||||
function getAlertDedupeKey(kind: AlertPayload["kind"]): string {
|
||||
return `lolympus-agent:alert:dedupe:${kind}`;
|
||||
}
|
||||
|
||||
export class AlertService implements AlertSender {
|
||||
private readonly logger = getLogger();
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "setStringIfAbsent">,
|
||||
private readonly natsClient: Pick<NatsClient, "publish">,
|
||||
) {}
|
||||
|
||||
async sendAlert(
|
||||
payload: AlertPayload,
|
||||
options?: {
|
||||
dedupeKey?: string | undefined;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
const dedupeKey = options?.dedupeKey ?? getAlertDedupeKey(payload.kind);
|
||||
const claimed = await this.garnetClient.setStringIfAbsent(
|
||||
dedupeKey,
|
||||
payload.timestamp,
|
||||
getAlertRateLimitSec(),
|
||||
);
|
||||
|
||||
if (!claimed) {
|
||||
this.logger.debug(
|
||||
{
|
||||
dedupeKey,
|
||||
kind: payload.kind,
|
||||
},
|
||||
"[Alert] Suppressed by rate limit",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
await this.natsClient.publish(AGENT_ALERT_SUBJECT, payload);
|
||||
this.logger.warn(
|
||||
{
|
||||
kind: payload.kind,
|
||||
metric: payload.metric,
|
||||
severity: payload.severity,
|
||||
threshold: payload.threshold,
|
||||
value: payload.value,
|
||||
window: payload.window,
|
||||
},
|
||||
"[Alert] Published",
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -17,11 +17,18 @@ export function initTracing(): void {
|
||||
return;
|
||||
}
|
||||
|
||||
const exporterType = getOtelExporter();
|
||||
if (exporterType === 'none') {
|
||||
const logger = getLogger();
|
||||
logger.debug('[Tracing] Exporter set to "none", skipping OTel SDK init');
|
||||
started = true;
|
||||
return;
|
||||
}
|
||||
|
||||
process.env['OTEL_SERVICE_NAME'] ??= getServiceName();
|
||||
|
||||
const logger = getLogger();
|
||||
const exporter =
|
||||
getOtelExporter() === 'stdout' ? new ConsoleSpanExporter() : new ConsoleSpanExporter();
|
||||
const exporter = new ConsoleSpanExporter();
|
||||
|
||||
sdk = new NodeSDK({
|
||||
instrumentations: [getNodeAutoInstrumentations()],
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
export const VALID_TIERS = [
|
||||
"GM",
|
||||
"I",
|
||||
"B",
|
||||
"S",
|
||||
"G",
|
||||
"P",
|
||||
"E",
|
||||
"D",
|
||||
"M",
|
||||
"C",
|
||||
"U",
|
||||
] as const;
|
||||
|
||||
export const VALID_TIER_SET = new Set<string>(VALID_TIERS);
|
||||
export const VALID_GENDERS = ["남", "여"] as const;
|
||||
export const VALID_GENDER_SET = new Set<string>(VALID_GENDERS);
|
||||
|
||||
export const MIN_AGE = 27;
|
||||
export const MAX_AGE: number | null = null;
|
||||
export const NICKNAME_FORMAT_EXAMPLE = "홍길동/95/남/G" as const;
|
||||
export const TIER_CODES_TEXT = VALID_TIERS.join(", ");
|
||||
|
||||
export const NICKNAME_FORMAT_TEXT =
|
||||
`닉네임은 이름/출생 연도 두 자리/성별/티어 형식이에요. 예를 들면 ${NICKNAME_FORMAT_EXAMPLE}처럼 쓰시면 돼요.`;
|
||||
|
||||
export const HEART_RULE_TEXT =
|
||||
'채팅방 오른쪽 상단 메뉴 버튼에서 "커버 보기"로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.';
|
||||
|
||||
export const HEART_LOCATION_TEXT =
|
||||
'채팅방 오른쪽 상단 메뉴 버튼을 누르면 "커버 보기"가 보여요. 거기서 프로필과 하트를 확인하실 수 있어요.';
|
||||
|
||||
export const DISCORD_GUIDE_TEXT =
|
||||
"닉네임 변경과 하트 인증을 모두 완료하신 뒤에 디스코드 서버에서 프로필을 설정하시면 권한이 자동으로 부여돼요.";
|
||||
|
||||
export const DISCORD_INVITE_URL =
|
||||
process.env["DISCORD_INVITE_URL"]?.trim() || "";
|
||||
|
||||
export const DEADLINE_NEAR_THRESHOLD_MIN = Number(
|
||||
process.env["DEADLINE_NEAR_THRESHOLD_MIN"] ?? 30,
|
||||
);
|
||||
|
||||
export const ESCALATION_BUDGET_PER_TURN = Number(
|
||||
process.env["ESCALATION_BUDGET_PER_TURN"] ?? 2,
|
||||
);
|
||||
@@ -1,12 +1,19 @@
|
||||
import type { NicknameIssue, OnboardingSession } from "./types.js";
|
||||
import {
|
||||
HEART_LOCATION_TEXT,
|
||||
HEART_RULE_TEXT,
|
||||
MIN_AGE,
|
||||
NICKNAME_FORMAT_EXAMPLE,
|
||||
NICKNAME_FORMAT_TEXT,
|
||||
TIER_CODES_TEXT,
|
||||
VALID_GENDER_SET,
|
||||
VALID_TIER_SET,
|
||||
} from "./constants.js";
|
||||
|
||||
function pickRandom(items: [string, ...string[]]): string {
|
||||
return items[Math.floor(Math.random() * items.length)] ?? items[0];
|
||||
}
|
||||
|
||||
const VALID_GENDERS = new Set(["남", "여"]);
|
||||
const VALID_TIERS = new Set(["GM", "I", "B", "S", "G", "P", "E", "D", "M", "C", "U"]);
|
||||
|
||||
export function analyzeNicknameIssues(nickname: string): NicknameIssue[] {
|
||||
const parts = nickname.trim().split("/");
|
||||
const issues: NicknameIssue[] = [];
|
||||
@@ -21,9 +28,9 @@ export function analyzeNicknameIssues(nickname: string): NicknameIssue[] {
|
||||
});
|
||||
}
|
||||
if (third) {
|
||||
const hasGender = VALID_GENDERS.has(third.charAt(0));
|
||||
const hasGender = VALID_GENDER_SET.has(third.charAt(0));
|
||||
const rest = third.slice(1);
|
||||
if (hasGender && rest.length > 0 && !VALID_TIERS.has(rest)) {
|
||||
if (hasGender && rest.length > 0 && !VALID_TIER_SET.has(rest)) {
|
||||
issues.push({
|
||||
kind: "unknown-tier",
|
||||
message: `성별 뒤의 '${rest}'를 제거하고, 티어 코드를 '/'로 구분해서 추가해주세요 (예: ${third.charAt(0)}/D)`,
|
||||
@@ -74,17 +81,17 @@ export function analyzeNicknameIssues(nickname: string): NicknameIssue[] {
|
||||
});
|
||||
}
|
||||
|
||||
if (!gender || !VALID_GENDERS.has(gender)) {
|
||||
if (!gender || !VALID_GENDER_SET.has(gender)) {
|
||||
issues.push({
|
||||
kind: "missing-gender",
|
||||
message: "성별을 '남' 또는 '여'로 입력해주세요",
|
||||
});
|
||||
}
|
||||
|
||||
if (!tier || !VALID_TIERS.has(tier)) {
|
||||
if (!tier || !VALID_TIER_SET.has(tier)) {
|
||||
issues.push({
|
||||
kind: "unknown-tier",
|
||||
message: "티어 코드(GM,I,B,S,G,P,E,D,M,C,U 중 하나)를 입력해주세요",
|
||||
message: `티어 코드(${TIER_CODES_TEXT} 중 하나)를 입력해주세요`,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -146,29 +153,29 @@ export function getNicknameInvalidMessage(
|
||||
|
||||
export function getNicknameFormatMessage(): string {
|
||||
return pickRandom([
|
||||
"닉네임 형식은 이름/출생 연도 두 자리/성별/티어예요. 예를 들면 홍길동/95/여/D처럼 쓰시면 돼요.",
|
||||
"닉네임은 이름/출생 연도 두 자리/성별/티어 형식으로 입력해 주시면 돼요! 예: 홍길동/95/남/G",
|
||||
NICKNAME_FORMAT_TEXT,
|
||||
`닉네임은 이름/출생 연도 두 자리/성별/티어 형식으로 입력해 주시면 돼요! 예: ${NICKNAME_FORMAT_EXAMPLE}`,
|
||||
]);
|
||||
}
|
||||
|
||||
export function getHeartGuideMessage(): string {
|
||||
return pickRandom([
|
||||
'채팅방 오른쪽 상단 메뉴 버튼에서 "커버 보기"로 들어가 하트를 빨간색으로 바꾼 뒤, 편집 없이 원본 스크린샷을 올려주시면 돼요.',
|
||||
HEART_RULE_TEXT,
|
||||
"채팅방 프로필에서 하트를 눌러 빨간색으로 만든 다음, 그 화면이 보이게 원본 스크린샷을 보내주시면 인증돼요.",
|
||||
]);
|
||||
}
|
||||
|
||||
export function getHeartLocationMessage(): string {
|
||||
return pickRandom([
|
||||
'채팅방 오른쪽 상단 메뉴 버튼을 누르면 "커버 보기"가 보여요. 거기서 프로필과 하트를 확인하실 수 있어요.',
|
||||
HEART_LOCATION_TEXT,
|
||||
'오른쪽 상단 메뉴 버튼으로 들어가 "커버 보기"를 열면 하트 위치를 확인하실 수 있어요.',
|
||||
]);
|
||||
}
|
||||
|
||||
export function getAgeRequirementMessage(): string {
|
||||
return pickRandom([
|
||||
"롤림푸스는 27세 이상만 가입이 가능해요.",
|
||||
"가입 가능 연령은 27세 이상이에요.",
|
||||
`롤림푸스는 ${MIN_AGE}세 이상만 가입이 가능해요.`,
|
||||
`가입 가능 연령은 ${MIN_AGE}세 이상이에요.`,
|
||||
]);
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
policyLayer,
|
||||
stateLayer,
|
||||
} from "./layers.js";
|
||||
import { buildToolPolicy } from "./tool-policy.js";
|
||||
|
||||
export class PromptBuilder {
|
||||
private constructor(
|
||||
@@ -35,7 +36,12 @@ export class PromptBuilder {
|
||||
|
||||
const systemPrompt = [
|
||||
personaLayer(),
|
||||
policyLayer(this.derived.intent),
|
||||
policyLayer(this.derived.intent, {
|
||||
toolPolicy:
|
||||
this.derived.responseMode === "llm-with-tools"
|
||||
? buildToolPolicy(this.derived.intent.kind, this.derived.phase.kind)
|
||||
: undefined,
|
||||
}),
|
||||
stateLayer(this.session, this.derived),
|
||||
examplesLayer(this.derived.intent),
|
||||
contextLayer(promptContext),
|
||||
|
||||
@@ -7,6 +7,12 @@ import type {
|
||||
OnboardingSession,
|
||||
PromptContext,
|
||||
} from "../types.js";
|
||||
import {
|
||||
HEART_RULE_TEXT,
|
||||
MIN_AGE,
|
||||
NICKNAME_FORMAT_EXAMPLE,
|
||||
TIER_CODES_TEXT,
|
||||
} from "../constants.js";
|
||||
import { EXAMPLES_BY_INTENT } from "./examples.js";
|
||||
|
||||
function formatKstTimestamp(value: string | undefined): string {
|
||||
@@ -94,11 +100,11 @@ export function personaLayer(): string {
|
||||
처음 온 멤버가 편안하게 등록을 마칠 수 있도록 자연스럽고 정확하게 안내해줘.
|
||||
|
||||
[커뮤니티 규칙]
|
||||
- 가입 가능 연령은 27세 이상이야.
|
||||
- 가입 가능 연령은 ${MIN_AGE}세 이상이야.
|
||||
- 새내기 등록은 닉네임 변경과 방 하트 인증 두 가지를 모두 완료하면 돼.
|
||||
- 닉네임 형식은 이름/출생 연도 두 자리/성별/티어야. 예: 홍길동/95/남/G
|
||||
- 성별은 남 또는 여, 티어 코드는 GM, I, B, S, G, P, E, D, M, C, U 중 하나야.
|
||||
- 방 하트 인증은 채팅방 프로필에서 하트를 빨간색으로 바꾼 뒤 편집 없는 원본 스크린샷을 보내는 절차야.
|
||||
- 닉네임 형식은 이름/출생 연도 두 자리/성별/티어야. 예: ${NICKNAME_FORMAT_EXAMPLE}
|
||||
- 성별은 남 또는 여, 티어 코드는 ${TIER_CODES_TEXT} 중 하나야.
|
||||
- 방 하트 인증은 ${HEART_RULE_TEXT}
|
||||
- 디스코드 안내는 닉네임 변경과 하트 인증이 모두 완료된 뒤에만 해줘.`;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import type { Intent, OnboardingPhase } from "../state/onboarding-state.js";
|
||||
|
||||
export function buildToolPolicy(
|
||||
intent: Intent["kind"],
|
||||
phase: OnboardingPhase["kind"],
|
||||
): string {
|
||||
if (intent === "nickname-question" && phase === "nickname-rejected") {
|
||||
return "- 닉네임 후보나 현재 닉네임을 설명할 때는 validate-nickname 도구를 먼저 사용한 뒤, 틀린 부분만 짚어.";
|
||||
}
|
||||
|
||||
if (intent === "status-question") {
|
||||
return "- 진행 상태 질문은 check-onboarding-progress 도구를 우선 사용하고, 상태만 짧게 답해.";
|
||||
}
|
||||
|
||||
if (intent === "age-question" || intent === "discord-question") {
|
||||
return "- 규칙 확인이 필요하면 lookup-community-rule 도구를 사용해 정확한 규칙만 답해.";
|
||||
}
|
||||
|
||||
if (intent === "help-where") {
|
||||
return "- 위치 안내가 모호할 때만 get-room-context 도구를 고려하고, 기본적으로는 절차 위치만 답해.";
|
||||
}
|
||||
|
||||
if (phase === "deadline-near") {
|
||||
return "- 마감이 가까운 상태에서는 남은 절차만 우선순위로 짧게 답해.";
|
||||
}
|
||||
|
||||
return "- 도구가 꼭 필요하지 않다면 호출하지 말고, 지금 질문에 대한 답만 간결하게 해.";
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import {
|
||||
getCriticMaxTokens,
|
||||
getLlmTemperature,
|
||||
getTier2CriticModel,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
export interface CritiqueResult {
|
||||
changed: boolean;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function serializeMessages(messages: ChatMessage[]): string {
|
||||
return messages
|
||||
.map((message) => `[${message.role}] ${message.content}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function buildCriticMessages(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): ChatMessage[] {
|
||||
return [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"당신은 온보딩 응답을 검토하는 critic입니다.",
|
||||
"초안이 충분히 정확하고 범위를 벗어나지 않으며 간결하면 반드시 [OK]만 출력하세요.",
|
||||
"고쳐야 한다면 유저에게 바로 보낼 최종 한국어 답변만 출력하고, 설명이나 해설은 쓰지 마세요.",
|
||||
"질문에서 묻지 않은 추가 절차를 길게 덧붙이지 말고, 꼭 필요한 내용만 남기세요.",
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}`,
|
||||
"[CONVERSATION]",
|
||||
serializeMessages(messages),
|
||||
"[DRAFT]",
|
||||
draft,
|
||||
].join("\n\n"),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
export class OnboardingCritic {
|
||||
constructor(private readonly llmClient: LlmClient) {}
|
||||
|
||||
async critique(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options?: { deadlineMs?: number | undefined },
|
||||
): Promise<CritiqueResult> {
|
||||
const response = await this.llmClient.chatCompletion(
|
||||
buildCriticMessages(draft, derived, messages),
|
||||
{
|
||||
deadlineMs: options?.deadlineMs,
|
||||
maxTokens: getCriticMaxTokens(),
|
||||
modelOverride: getTier2CriticModel(),
|
||||
role: "critic",
|
||||
temperature: Math.max(0, getLlmTemperature() - 0.15),
|
||||
tier: 2,
|
||||
},
|
||||
);
|
||||
|
||||
const text = response?.content.trim() ?? "";
|
||||
if (!text || text.startsWith("[OK]")) {
|
||||
return {
|
||||
changed: false,
|
||||
text: draft,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
changed: true,
|
||||
text,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { OnboardingOrchestrator } from "./orchestrator.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
function createDerivedState(overrides?: Partial<DerivedState>): DerivedState {
|
||||
return {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "help-how" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 256,
|
||||
allowShortResponse: false,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("OnboardingOrchestrator", () => {
|
||||
it("uses direct response path for tier 0", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
return Promise.resolve({
|
||||
content: "안내 도와드릴게요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "안내 도와드릴게요." } }] },
|
||||
message: { role: "assistant" as const, content: "안내 도와드릴게요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 0 }),
|
||||
[{ role: "user", content: "도와줘" }],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("안내 도와드릴게요.");
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]?.options?.role).toBe("responder");
|
||||
});
|
||||
|
||||
it("runs planner then responder for tier 1", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "redirect-to-step",
|
||||
targetPhase: "photo-pending",
|
||||
mustInclude: ["하트 인증만 안내"],
|
||||
mustAvoid: ["디스코드 절차"],
|
||||
suggestedTools: [],
|
||||
tone: "friendly",
|
||||
targetLengthChars: 120,
|
||||
rationale: "질문 범위만 답하기",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
|
||||
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 1 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "이제 뭐 남았지" },
|
||||
],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("하트 인증만 남았어요.");
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls[0]?.options?.role).toBe("planner");
|
||||
expect(calls[1]?.messages[0]?.content).toContain("[PLAN]");
|
||||
});
|
||||
|
||||
it("runs critic after responder for tier 2 and applies rewrite", async () => {
|
||||
const calls: Array<{
|
||||
messages: ChatMessage[];
|
||||
options: ChatCompletionOptions | undefined;
|
||||
}> = [];
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: ChatMessage[],
|
||||
options?: ChatCompletionOptions,
|
||||
) => {
|
||||
calls.push({ messages, options });
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: JSON.stringify({
|
||||
approach: "answer-direct",
|
||||
targetPhase: "photo-pending",
|
||||
mustInclude: [],
|
||||
mustAvoid: ["추가 절차 나열"],
|
||||
suggestedTools: [],
|
||||
tone: "concise",
|
||||
targetLengthChars: 60,
|
||||
rationale: "짧게 답하기",
|
||||
}),
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "{}" } }] },
|
||||
message: { role: "assistant" as const, content: "{}" },
|
||||
});
|
||||
}
|
||||
|
||||
if (calls.length === 2) {
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: {
|
||||
choices: [
|
||||
{ message: { content: "하트 인증만 남았어요. 커버 보기로 들어가서..." } },
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant" as const,
|
||||
content: "하트 인증만 남았어요. 커버 보기로 들어가서...",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "하트 인증만 남았어요.",
|
||||
endpoint: "mock://llm",
|
||||
provider: "cliproxy" as const,
|
||||
raw: { choices: [{ message: { content: "하트 인증만 남았어요." } }] },
|
||||
message: { role: "assistant" as const, content: "하트 인증만 남았어요." },
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const orchestrator = new OnboardingOrchestrator(llmClientStub as never);
|
||||
const result = await orchestrator.run(
|
||||
createDerivedState({ tier: 2 }),
|
||||
[
|
||||
{ role: "system", content: "기본 시스템 프롬프트" },
|
||||
{ role: "user", content: "이제 뭐 남았지" },
|
||||
],
|
||||
{ toolContext: {} },
|
||||
);
|
||||
|
||||
expect(result).toBe("하트 인증만 남았어요.");
|
||||
expect(calls).toHaveLength(3);
|
||||
expect(calls[2]?.options?.role).toBe("critic");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,382 @@
|
||||
import { getContext } from "@obs/context";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import {
|
||||
getLlmTemperature,
|
||||
getPlannerMaxTokens,
|
||||
getTier0EstimatedCallMs,
|
||||
getTier0MaxTokens,
|
||||
getTier0Model,
|
||||
getTier1EstimatedCallMs,
|
||||
getTier1MaxTokens,
|
||||
getTier1PlannerModel,
|
||||
getTier1ResponderModel,
|
||||
getTier2EstimatedCallMs,
|
||||
getTier2MaxTokens,
|
||||
} from "../../../config.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
import { OnboardingCritic } from "./critic.js";
|
||||
import type { PlannerOutput } from "./planner-schema.js";
|
||||
import { PlannerOutputSchema } from "./planner-schema.js";
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
|
||||
export interface OrchestratorRunOptions {
|
||||
maxTokens?: number | undefined;
|
||||
temperature?: number | undefined;
|
||||
toolContext: ToolContext;
|
||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||
}
|
||||
|
||||
function cloneMessages(messages: ChatMessage[]): ChatMessage[] {
|
||||
return messages.map((message) => ({
|
||||
...message,
|
||||
tool_calls: message.tool_calls ? [...message.tool_calls] : undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
function injectSystemAddendum(
|
||||
messages: ChatMessage[],
|
||||
addendum: string,
|
||||
): ChatMessage[] {
|
||||
const cloned = cloneMessages(messages);
|
||||
const firstSystemIndex = cloned.findIndex((message) => message.role === "system");
|
||||
|
||||
if (firstSystemIndex === -1) {
|
||||
cloned.unshift({
|
||||
role: "system",
|
||||
content: addendum,
|
||||
});
|
||||
return cloned;
|
||||
}
|
||||
|
||||
const current = cloned[firstSystemIndex];
|
||||
if (!current) {
|
||||
return cloned;
|
||||
}
|
||||
|
||||
cloned[firstSystemIndex] = {
|
||||
...current,
|
||||
content: `${current.content}\n\n${addendum}`,
|
||||
};
|
||||
return cloned;
|
||||
}
|
||||
|
||||
function summarizeMessages(messages: ChatMessage[]): string {
|
||||
return messages
|
||||
.map((message) => `[${message.role}] ${message.content}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function tryParsePlannerOutput(raw: string): PlannerOutput | null {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return PlannerOutputSchema.parse(parsed);
|
||||
} catch {
|
||||
const start = raw.indexOf("{");
|
||||
const end = raw.lastIndexOf("}");
|
||||
if (start === -1 || end === -1 || end <= start) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw.slice(start, end + 1)) as unknown;
|
||||
return PlannerOutputSchema.parse(parsed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getDefaultPlan(derived: DerivedState): PlannerOutput {
|
||||
return {
|
||||
approach:
|
||||
derived.responseMode === "llm-with-tools"
|
||||
? "validate-then-explain"
|
||||
: derived.intent.kind === "help-how" || derived.intent.kind === "help-where"
|
||||
? "redirect-to-step"
|
||||
: "answer-direct",
|
||||
targetPhase: derived.phase.kind,
|
||||
mustInclude: [],
|
||||
mustAvoid: [],
|
||||
suggestedTools:
|
||||
derived.responseMode === "llm-with-tools" ? ["validate-nickname"] : [],
|
||||
tone: derived.allowShortResponse ? "concise" : "friendly",
|
||||
targetLengthChars: derived.allowShortResponse ? 80 : 150,
|
||||
rationale: "fallback-plan",
|
||||
};
|
||||
}
|
||||
|
||||
function getRemainingDeadlineMs(): number | undefined {
|
||||
const deadlineAt = getContext()?.deadlineAt;
|
||||
if (!deadlineAt) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const deadlineMs = Date.parse(deadlineAt) - Date.now();
|
||||
return Number.isFinite(deadlineMs) ? deadlineMs : undefined;
|
||||
}
|
||||
|
||||
function canAffordCall(estimatedCallMs: number): boolean {
|
||||
const remainingMs = getRemainingDeadlineMs();
|
||||
return remainingMs === undefined || remainingMs > estimatedCallMs;
|
||||
}
|
||||
|
||||
function buildPlannerMessages(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): ChatMessage[] {
|
||||
return [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"당신은 온보딩 응답을 설계하는 planner입니다.",
|
||||
"반드시 JSON만 출력하세요.",
|
||||
"질문에 직접 답할 수 있으면 answer-direct를 우선하고, 유저가 묻지 않은 추가 절차는 mustAvoid에 넣으세요.",
|
||||
"tool은 꼭 필요한 경우에만 suggestedTools에 넣고 최대 3개까지만 고르세요.",
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`[STATE] intent=${derived.intent.kind}, phase=${derived.phase.kind}, responseMode=${derived.responseMode}, tier=${derived.tier}`,
|
||||
"[CONVERSATION]",
|
||||
summarizeMessages(messages),
|
||||
"[SCHEMA]",
|
||||
JSON.stringify({
|
||||
approach:
|
||||
"answer-direct | ask-clarification | redirect-to-step | validate-then-explain | lookup-rule-then-answer | soft-decline",
|
||||
targetPhase:
|
||||
"fresh | nickname-pending | photo-pending | both-pending | nickname-rejected | photo-rejected | deadline-near | completed | timed-out",
|
||||
mustInclude: ["short strings"],
|
||||
mustAvoid: ["short strings"],
|
||||
suggestedTools: [
|
||||
"validate-nickname | check-onboarding-progress | lookup-community-rule | get-room-context",
|
||||
],
|
||||
tone: "friendly | concise | encouraging | neutral | apologetic",
|
||||
targetLengthChars: 150,
|
||||
rationale: "short reason",
|
||||
}),
|
||||
].join("\n\n"),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function buildResponderAddendum(plan: PlannerOutput): string {
|
||||
const lines = [
|
||||
"[PLAN]",
|
||||
`approach=${plan.approach}`,
|
||||
`tone=${plan.tone}`,
|
||||
`targetLengthChars=${plan.targetLengthChars}`,
|
||||
];
|
||||
|
||||
if (plan.targetPhase) {
|
||||
lines.push(`targetPhase=${plan.targetPhase}`);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
`mustInclude=${plan.mustInclude.length > 0 ? plan.mustInclude.join(" | ") : "-"}`,
|
||||
`mustAvoid=${plan.mustAvoid.length > 0 ? plan.mustAvoid.join(" | ") : "-"}`,
|
||||
"위 계획을 참고하되, 실제 유저에게 보낼 최종 한국어 답변만 출력하세요.",
|
||||
);
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function pickToolsForPlan(
|
||||
plan: PlannerOutput,
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
): readonly Tool<unknown, unknown>[] {
|
||||
if (plan.approach === "answer-direct" || plan.approach === "soft-decline") {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (plan.suggestedTools.length === 0) {
|
||||
return tools;
|
||||
}
|
||||
|
||||
const picked = tools.filter((tool) => plan.suggestedTools.includes(tool.name as never));
|
||||
return picked.length > 0 ? picked : tools;
|
||||
}
|
||||
|
||||
export class OnboardingOrchestrator {
|
||||
private readonly critic: OnboardingCritic;
|
||||
|
||||
constructor(private readonly llmClient: LlmClient) {
|
||||
this.critic = new OnboardingCritic(llmClient);
|
||||
}
|
||||
|
||||
private async runDirect(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
responderTools: readonly Tool<unknown, unknown>[],
|
||||
maxTokens: number,
|
||||
modelOverride: string,
|
||||
): Promise<string | null> {
|
||||
const deadlineMs = getRemainingDeadlineMs();
|
||||
|
||||
if (responderTools.length > 0) {
|
||||
return runWithTools({
|
||||
deadlineMs,
|
||||
llmClient: this.llmClient,
|
||||
maxTokens,
|
||||
messages,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
tier: derived.tier,
|
||||
temperature: options.temperature ?? getLlmTemperature(),
|
||||
toolContext: options.toolContext,
|
||||
tools: responderTools,
|
||||
});
|
||||
}
|
||||
|
||||
const response = await this.llmClient.chatCompletion(messages, {
|
||||
deadlineMs,
|
||||
maxTokens,
|
||||
modelOverride,
|
||||
role: "responder",
|
||||
temperature: options.temperature ?? getLlmTemperature(),
|
||||
tier: derived.tier,
|
||||
});
|
||||
|
||||
return response?.content ?? null;
|
||||
}
|
||||
|
||||
private async runPlanner(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
): Promise<PlannerOutput> {
|
||||
const plannerMessages = buildPlannerMessages(derived, messages);
|
||||
const firstAttempt = await this.llmClient.chatCompletion(plannerMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
jsonMode: true,
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
temperature: 0,
|
||||
tier: 1,
|
||||
});
|
||||
|
||||
const firstPlan = firstAttempt?.content
|
||||
? tryParsePlannerOutput(firstAttempt.content)
|
||||
: null;
|
||||
if (firstPlan) {
|
||||
return firstPlan;
|
||||
}
|
||||
|
||||
const retryMessages = [
|
||||
...plannerMessages,
|
||||
{
|
||||
role: "user" as const,
|
||||
content:
|
||||
"방금 응답은 스키마를 벗어났어요. 설명 없이 JSON object 하나만 다시 출력하세요.",
|
||||
},
|
||||
];
|
||||
|
||||
const retry = await this.llmClient.chatCompletion(retryMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
jsonMode: true,
|
||||
maxTokens: getPlannerMaxTokens(),
|
||||
modelOverride: getTier1PlannerModel(),
|
||||
role: "planner",
|
||||
temperature: 0,
|
||||
tier: 1,
|
||||
});
|
||||
|
||||
const retryPlan = retry?.content ? tryParsePlannerOutput(retry.content) : null;
|
||||
return retryPlan ?? getDefaultPlan(derived);
|
||||
}
|
||||
|
||||
private async runTier0(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
const maxTokens = options.maxTokens ?? getTier0MaxTokens();
|
||||
const tools =
|
||||
derived.responseMode === "llm-with-tools" ? (options.tools ?? []) : [];
|
||||
|
||||
return this.runDirect(
|
||||
derived,
|
||||
messages,
|
||||
options,
|
||||
tools,
|
||||
maxTokens,
|
||||
getTier0Model(),
|
||||
);
|
||||
}
|
||||
|
||||
private async runTier1(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
const plan = await this.runPlanner(derived, messages);
|
||||
const responderMessages = injectSystemAddendum(
|
||||
messages,
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
|
||||
|
||||
return this.runDirect(
|
||||
derived,
|
||||
responderMessages,
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier1MaxTokens(),
|
||||
getTier1ResponderModel(),
|
||||
);
|
||||
}
|
||||
|
||||
private async runTier2(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
const plan = await this.runPlanner(derived, messages);
|
||||
const responderMessages = injectSystemAddendum(
|
||||
messages,
|
||||
buildResponderAddendum(plan),
|
||||
);
|
||||
const responderTools = pickToolsForPlan(plan, options.tools ?? []);
|
||||
|
||||
const draft = await this.runDirect(
|
||||
derived,
|
||||
responderMessages,
|
||||
options,
|
||||
responderTools,
|
||||
options.maxTokens ?? getTier2MaxTokens(),
|
||||
getTier1ResponderModel(),
|
||||
);
|
||||
|
||||
if (!draft) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const critique = await this.critic.critique(draft, derived, responderMessages, {
|
||||
deadlineMs: getRemainingDeadlineMs(),
|
||||
});
|
||||
return critique.text;
|
||||
}
|
||||
|
||||
async run(
|
||||
derived: DerivedState,
|
||||
messages: ChatMessage[],
|
||||
options: OrchestratorRunOptions,
|
||||
): Promise<string | null> {
|
||||
if (derived.tier >= 2 && canAffordCall(getTier2EstimatedCallMs())) {
|
||||
return this.runTier2(derived, messages, options);
|
||||
}
|
||||
|
||||
if (derived.tier >= 1 && canAffordCall(getTier1EstimatedCallMs())) {
|
||||
return this.runTier1(derived, messages, options);
|
||||
}
|
||||
|
||||
if (!canAffordCall(getTier0EstimatedCallMs())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.runTier0(derived, messages, options);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const PlannerApproachSchema = z.enum([
|
||||
"answer-direct",
|
||||
"ask-clarification",
|
||||
"redirect-to-step",
|
||||
"validate-then-explain",
|
||||
"lookup-rule-then-answer",
|
||||
"soft-decline",
|
||||
]);
|
||||
|
||||
export const PlannerTargetPhaseSchema = z.enum([
|
||||
"fresh",
|
||||
"nickname-pending",
|
||||
"photo-pending",
|
||||
"both-pending",
|
||||
"nickname-rejected",
|
||||
"photo-rejected",
|
||||
"deadline-near",
|
||||
"completed",
|
||||
"timed-out",
|
||||
]);
|
||||
|
||||
export const PlannerSuggestedToolSchema = z.enum([
|
||||
"validate-nickname",
|
||||
"check-onboarding-progress",
|
||||
"lookup-community-rule",
|
||||
"get-room-context",
|
||||
]);
|
||||
|
||||
export const PlannerToneSchema = z.enum([
|
||||
"friendly",
|
||||
"concise",
|
||||
"encouraging",
|
||||
"neutral",
|
||||
"apologetic",
|
||||
]);
|
||||
|
||||
export const PlannerOutputSchema = z.object({
|
||||
approach: PlannerApproachSchema,
|
||||
targetPhase: PlannerTargetPhaseSchema.optional(),
|
||||
mustInclude: z.array(z.string().min(1).max(80)).max(5).default([]),
|
||||
mustAvoid: z.array(z.string().min(1).max(80)).max(5).default([]),
|
||||
suggestedTools: z.array(PlannerSuggestedToolSchema).max(3).default([]),
|
||||
tone: PlannerToneSchema.default("friendly"),
|
||||
targetLengthChars: z.number().int().min(20).max(500).default(150),
|
||||
rationale: z.string().max(200),
|
||||
});
|
||||
|
||||
export type PlannerOutput = z.infer<typeof PlannerOutputSchema>;
|
||||
@@ -0,0 +1,219 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import { runWithTools } from "./tool-loop.js";
|
||||
import type { Tool } from "@tools/tool";
|
||||
|
||||
describe("runWithTools", () => {
|
||||
it("executes a tool call and returns the model's final answer", async () => {
|
||||
const calls: Array<{
|
||||
content: string;
|
||||
messages: Array<{ role: string; content: string }>;
|
||||
options?: Record<string, unknown> | undefined;
|
||||
}> = [];
|
||||
|
||||
const llmClientStub = {
|
||||
chatCompletion: (
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
options?: Record<string, unknown>,
|
||||
) => {
|
||||
calls.push({
|
||||
content: messages[messages.length - 1]?.content ?? "",
|
||||
messages,
|
||||
options,
|
||||
});
|
||||
|
||||
if (calls.length === 1) {
|
||||
return Promise.resolve({
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content:
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여D"}}</tool_call>',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
||||
endpoint: "mock://llm",
|
||||
provider: "llama",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "티어는 '/'로 구분해서 넣어주시면 돼요. 예: 프리리/95/여/D",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ nickname: string }, { valid: boolean; issues: string[] }> = {
|
||||
name: "validate-nickname",
|
||||
description: "닉네임 형식 검사",
|
||||
argsSchema: z.object({
|
||||
nickname: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
valid: z.boolean(),
|
||||
issues: z.array(z.string()),
|
||||
}),
|
||||
execute: () =>
|
||||
Promise.resolve({
|
||||
valid: false,
|
||||
issues: ["티어는 '/'로 구분해서 넣어주세요."],
|
||||
}),
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "도구를 필요할 때만 사용해.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "닉네임이 왜 안 돼요?",
|
||||
},
|
||||
],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toContain("프리리/95/여/D");
|
||||
expect(calls[0]?.options?.tools).toHaveLength(1);
|
||||
expect(calls[1]?.messages.some((message) => message.role === "tool")).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves native tool call responses for follow-up turns", async () => {
|
||||
const seenMessages: Array<Array<Record<string, unknown>>> = [];
|
||||
|
||||
const llmClientStub = {
|
||||
chatCompletion: (messages: Array<Record<string, unknown>>) => {
|
||||
seenMessages.push(messages);
|
||||
|
||||
if (seenMessages.length === 1) {
|
||||
return Promise.resolve({
|
||||
content: "",
|
||||
endpoint: "mock://cliproxy",
|
||||
provider: "cliproxy",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: {
|
||||
name: "lookup-community-rule",
|
||||
arguments: '{"topic":"age"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "",
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: {
|
||||
name: "lookup-community-rule",
|
||||
arguments: '{"topic":"age"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve({
|
||||
content: "나이 기준은 27세 이상이에요.",
|
||||
endpoint: "mock://cliproxy",
|
||||
provider: "cliproxy",
|
||||
raw: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: "나이 기준은 27세 이상이에요.",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "나이 기준은 27세 이상이에요.",
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const tool: Tool<{ topic: string }, { topic: string; text: string }> = {
|
||||
name: "lookup-community-rule",
|
||||
description: "커뮤니티 규칙 조회",
|
||||
argsSchema: z.object({
|
||||
topic: z.string(),
|
||||
}),
|
||||
resultSchema: z.object({
|
||||
topic: z.string(),
|
||||
text: z.string(),
|
||||
}),
|
||||
execute: () =>
|
||||
Promise.resolve({
|
||||
topic: "age",
|
||||
text: "나이 기준은 27세 이상이에요.",
|
||||
}),
|
||||
};
|
||||
|
||||
const result = await runWithTools({
|
||||
llmClient: llmClientStub as never,
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "필요할 때만 도구를 사용해.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "나이 기준이 뭐예요?",
|
||||
},
|
||||
],
|
||||
toolContext: {},
|
||||
tools: [tool],
|
||||
});
|
||||
|
||||
expect(result).toBe("나이 기준은 27세 이상이에요.");
|
||||
expect(seenMessages[1]?.some((message) => message.role === "tool")).toBe(true);
|
||||
expect(
|
||||
seenMessages[1]?.some(
|
||||
(message) => Array.isArray(message.tool_calls) && message.role === "assistant",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,89 @@
|
||||
import { withToolSpan } from "@obs/spans";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatCompletionOptions, ChatMessage } from "@llm/types";
|
||||
import { parseToolCalls } from "@tools/parser";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
|
||||
export interface ToolLoopOptions {
|
||||
deadlineMs?: number | undefined;
|
||||
llmClient: LlmClient;
|
||||
maxHops?: number | undefined;
|
||||
maxTokens?: number | undefined;
|
||||
messages: ChatMessage[];
|
||||
modelOverride?: string | undefined;
|
||||
role?: ChatCompletionOptions["role"];
|
||||
signal?: AbortSignal | undefined;
|
||||
temperature?: number | undefined;
|
||||
tier?: ChatCompletionOptions["tier"];
|
||||
toolContext: ToolContext;
|
||||
tools: readonly Tool<unknown, unknown>[];
|
||||
}
|
||||
|
||||
export async function runWithTools(
|
||||
options: ToolLoopOptions,
|
||||
): Promise<string | null> {
|
||||
const maxHops = options.maxHops ?? 2;
|
||||
const messages = options.messages.map((message) => ({ ...message }));
|
||||
let allowNativeTools = true;
|
||||
|
||||
for (let hop = 0; hop <= maxHops; hop++) {
|
||||
const response = await options.llmClient.chatCompletion(messages, {
|
||||
deadlineMs: options.deadlineMs,
|
||||
maxTokens: options.maxTokens,
|
||||
modelOverride: options.modelOverride,
|
||||
role: options.role,
|
||||
signal: options.signal,
|
||||
temperature: options.temperature,
|
||||
tier: options.tier,
|
||||
tools: allowNativeTools ? options.tools : undefined,
|
||||
});
|
||||
|
||||
if (!response) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const toolCalls = parseToolCalls(response.message);
|
||||
if (toolCalls.length === 0) {
|
||||
return response.content;
|
||||
}
|
||||
|
||||
if (hop === maxHops) {
|
||||
messages.push(response.message);
|
||||
messages.push({
|
||||
role: "user",
|
||||
content:
|
||||
"도구 호출은 여기까지 하고, 지금까지 얻은 정보만으로 한국어 문장으로 최종 답변해줘.",
|
||||
});
|
||||
allowNativeTools = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
messages.push(response.message);
|
||||
|
||||
for (const call of toolCalls) {
|
||||
const tool = options.tools.find((candidate) => candidate.name === call.name);
|
||||
if (!tool) {
|
||||
throw new Error(`Tool not registered for call: ${call.name}`);
|
||||
}
|
||||
|
||||
const validated = tool.argsSchema.parse(call.arguments);
|
||||
const result = await withToolSpan(
|
||||
tool.name,
|
||||
{
|
||||
"tool.call.id": call.id,
|
||||
"tool.name": tool.name,
|
||||
},
|
||||
() => tool.execute(validated, options.toolContext),
|
||||
);
|
||||
|
||||
messages.push({
|
||||
role: "tool",
|
||||
name: tool.name,
|
||||
tool_call_id: call.id,
|
||||
content: JSON.stringify(result),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { validateOnboardingResponse } from "./validator.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
function makeDerived(
|
||||
overrides?: Partial<DerivedState>,
|
||||
): DerivedState {
|
||||
return {
|
||||
phase: { kind: "photo-pending" },
|
||||
intent: { kind: "photo-question" },
|
||||
shouldRespond: true,
|
||||
responseMode: "llm",
|
||||
suggestedMaxTokens: 256,
|
||||
allowShortResponse: false,
|
||||
tier: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("validateOnboardingResponse", () => {
|
||||
it("rejects discord guidance before onboarding is completed", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"디스코드 서버에 들어가시면 돼요.",
|
||||
makeDerived(),
|
||||
);
|
||||
|
||||
expect(result.status).toBe("reject");
|
||||
expect(result.reasons).toContain("stateConsistency");
|
||||
});
|
||||
|
||||
it("rejects invalid nickname examples", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"홍길동/95/남/Z로 바꾸시면 돼요.",
|
||||
makeDerived({
|
||||
phase: { kind: "nickname-rejected", current: "홍길동/95/남/골드", issues: [] },
|
||||
intent: { kind: "nickname-question" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.status).toBe("reject");
|
||||
expect(result.reasons).toContain("nicknameExampleConsistency");
|
||||
});
|
||||
|
||||
it("soft-warns on repeated phrases", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"하트 인증 해주세요 하트 인증 해주세요 하트 인증 해주세요",
|
||||
makeDerived(),
|
||||
{ allowShortResponse: true },
|
||||
);
|
||||
|
||||
expect(result.status).toBe("soft-warn");
|
||||
expect(result.reasons).toContain("phraseRepetition");
|
||||
});
|
||||
|
||||
it("passes short status responses when allowShortResponse is enabled", () => {
|
||||
const result = validateOnboardingResponse(
|
||||
"이제 방 하트 인증만 남았어요.",
|
||||
makeDerived({
|
||||
intent: { kind: "status-question" },
|
||||
}),
|
||||
{ allowShortResponse: true },
|
||||
);
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { getMaxResponseLength } from "../../../config.js";
|
||||
import { VALID_TIER_SET } from "../constants.js";
|
||||
import { EXPECTED_KEYWORDS } from "../state/intent-keywords.js";
|
||||
import type { DerivedState } from "../state/onboarding-state.js";
|
||||
|
||||
const ALLOWED_URL_PATTERNS = [
|
||||
/discord\.gg\/lolympus/iu,
|
||||
/open\.kakao\.com\/o\/gOscBAEh/iu,
|
||||
];
|
||||
|
||||
export interface ValidationResult {
|
||||
status: "pass" | "soft-warn" | "reject";
|
||||
text: string;
|
||||
reasons: string[];
|
||||
}
|
||||
|
||||
function postProcessResponse(text: string): string {
|
||||
let processed = text.trim();
|
||||
|
||||
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
|
||||
|
||||
processed = processed.replace(
|
||||
/https?:\/\/[^\s)}\]]+/gu,
|
||||
(match) => {
|
||||
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
|
||||
return match;
|
||||
}
|
||||
return "";
|
||||
},
|
||||
);
|
||||
|
||||
processed = processed.replace(/ +/gu, " ").trim();
|
||||
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1");
|
||||
processed = processed.replace(/\*(.+?)\*/gu, "$1");
|
||||
processed = processed.replace(/__(.+?)__/gu, "$1");
|
||||
processed = processed.replace(/~~(.+?)~~/gu, "$1");
|
||||
processed = processed.replace(/^#{1,6}\s+/gmu, "");
|
||||
processed = processed.replace(/^[-*]\s+/gmu, "");
|
||||
processed = processed.replace(/`(.+?)`/gu, "$1");
|
||||
|
||||
const maxLen = getMaxResponseLength();
|
||||
if (processed.length > maxLen) {
|
||||
processed = processed.slice(0, maxLen).trimEnd();
|
||||
const sentenceEndPattern =
|
||||
/(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
|
||||
|
||||
for (let index = processed.length; index > maxLen * 0.5; index--) {
|
||||
const sub = processed.slice(0, index);
|
||||
if (sentenceEndPattern.test(sub)) {
|
||||
processed = sub;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return processed.trim();
|
||||
}
|
||||
|
||||
function findInvalidNicknameExamples(text: string): string[] {
|
||||
const matches = [
|
||||
...text.matchAll(/([^\s/]{1,20})\/(\d{2})\/(남|여)\/([A-Z]{1,2})/gu),
|
||||
];
|
||||
|
||||
return matches.flatMap((match) => {
|
||||
const tier = match[4];
|
||||
if (!tier || VALID_TIER_SET.has(tier)) {
|
||||
return [];
|
||||
}
|
||||
return [tier];
|
||||
});
|
||||
}
|
||||
|
||||
function hasIntentKeywordMatch(text: string, derived: DerivedState): boolean {
|
||||
const keywords = EXPECTED_KEYWORDS[derived.intent.kind];
|
||||
if (keywords.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const lower = text.toLowerCase();
|
||||
return keywords.some((keyword) => lower.includes(keyword.toLowerCase()));
|
||||
}
|
||||
|
||||
function hasRepeatedPhrase(text: string): boolean {
|
||||
const words = text.split(/\s+/u);
|
||||
if (words.length < 6) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const phrases = new Map<string, number>();
|
||||
for (let index = 0; index < words.length - 1; index++) {
|
||||
const bigram = `${words[index]} ${words[index + 1]}`;
|
||||
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
|
||||
}
|
||||
|
||||
return [...phrases.values()].some((count) => count >= 3);
|
||||
}
|
||||
|
||||
function markValidation(result: ValidationResult): ValidationResult {
|
||||
const activeSpan = trace.getActiveSpan();
|
||||
activeSpan?.setAttributes({
|
||||
"agent.validator.status": result.status,
|
||||
"agent.validator.warnings": result.reasons,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export function validateOnboardingResponse(
|
||||
draft: string,
|
||||
derived: DerivedState,
|
||||
options?: { allowShortResponse?: boolean | undefined },
|
||||
): ValidationResult {
|
||||
const text = postProcessResponse(draft);
|
||||
const rejectReasons: string[] = [];
|
||||
const softWarnReasons: string[] = [];
|
||||
|
||||
if (!text) {
|
||||
rejectReasons.push("empty");
|
||||
}
|
||||
|
||||
if (text.includes("[NO_RESPONSE]")) {
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text: "[NO_RESPONSE]",
|
||||
reasons: [],
|
||||
});
|
||||
}
|
||||
|
||||
if (!options?.allowShortResponse && text.length < 10) {
|
||||
rejectReasons.push("minLength");
|
||||
}
|
||||
|
||||
const mentionsDiscord = /(디스코드|discord|디코)/iu.test(text);
|
||||
const advancesDiscordStep = /(서버|프로필|권한|입장)/iu.test(text);
|
||||
const defersDiscordStep =
|
||||
/(먼저|완료.*후|완료하신 뒤|완료하신 후|모두 완료|완료되면)/u.test(text);
|
||||
|
||||
if (
|
||||
mentionsDiscord &&
|
||||
derived.phase.kind !== "completed" &&
|
||||
advancesDiscordStep &&
|
||||
!defersDiscordStep
|
||||
) {
|
||||
rejectReasons.push("stateConsistency");
|
||||
}
|
||||
|
||||
if (findInvalidNicknameExamples(text).length > 0) {
|
||||
rejectReasons.push("nicknameExampleConsistency");
|
||||
}
|
||||
|
||||
if (!hasIntentKeywordMatch(text, derived)) {
|
||||
softWarnReasons.push("intentAlignment");
|
||||
}
|
||||
|
||||
if (hasRepeatedPhrase(text)) {
|
||||
softWarnReasons.push("phraseRepetition");
|
||||
}
|
||||
|
||||
if (rejectReasons.length > 0) {
|
||||
return markValidation({
|
||||
status: "reject",
|
||||
text,
|
||||
reasons: [...rejectReasons, ...softWarnReasons],
|
||||
});
|
||||
}
|
||||
|
||||
if (softWarnReasons.length > 0) {
|
||||
return markValidation({
|
||||
status: "soft-warn",
|
||||
text,
|
||||
reasons: softWarnReasons,
|
||||
});
|
||||
}
|
||||
|
||||
return markValidation({
|
||||
status: "pass",
|
||||
text,
|
||||
reasons: [],
|
||||
});
|
||||
}
|
||||
+100
-91
@@ -1,11 +1,17 @@
|
||||
import type { AgentRequest, AgentResponse, ConversationMessage } from "@agent/types";
|
||||
import { getConversationSummaryKey } from "@conv/conversation-store";
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import { getContext, getRequestLogger } from "@obs/context";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import type { ChatMessage } from "@llm/types";
|
||||
import type { RoomContextService } from "@room/room-context-service";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import {
|
||||
type ShadowEvalPublisher,
|
||||
ShadowEvalBus,
|
||||
} from "@eval/shadow-bus";
|
||||
import { createBuiltInToolRegistry } from "@tools/builtins/index";
|
||||
import type { Tool, ToolContext } from "@tools/tool";
|
||||
import { PromptBuilder } from "./prompts/builder.js";
|
||||
import {
|
||||
getAcknowledgementMessage,
|
||||
@@ -23,103 +29,40 @@ import {
|
||||
getRemainingStatusMessage,
|
||||
getTimeoutWarningMessage,
|
||||
} from "./onboarding-templates.js";
|
||||
import { getOnboardingSessionKeyPrefix, getMaxResponseLength, getLlmTemperature } from "../../config.js";
|
||||
import { getOnboardingSessionKeyPrefix, getLlmTemperature } from "../../config.js";
|
||||
import { OnboardingBlackboardStore } from "./state/blackboard.js";
|
||||
import { deriveState } from "./state/derive.js";
|
||||
import { OnboardingIntentClassifier } from "./state/classifier.js";
|
||||
import type { DerivedState, Intent } from "./state/onboarding-state.js";
|
||||
import { OnboardingOrchestrator } from "./runtime/orchestrator.js";
|
||||
import { validateOnboardingResponse } from "./runtime/validator.js";
|
||||
import type { OnboardingSession, PromptContext } from "./types.js";
|
||||
|
||||
const ALLOWED_URL_PATTERNS = [
|
||||
/discord\.gg\/lolympus/u,
|
||||
/open\.kakao\.com\/o\/gOscBAEh/u,
|
||||
];
|
||||
|
||||
type LlmTextResult =
|
||||
| { status: "success"; text: string }
|
||||
| { status: "no_response" }
|
||||
| { status: "failed" };
|
||||
|
||||
function postProcessResponse(text: string): string {
|
||||
let processed = text.trim();
|
||||
|
||||
// Remove <think> blocks (Qwen thinking mode artifacts)
|
||||
processed = processed.replace(/<think>[\s\S]*?<\/think>/gu, "").trim();
|
||||
|
||||
// Strip URLs not in allowlist
|
||||
processed = processed.replace(
|
||||
/https?:\/\/[^\s)}\]]+/gu,
|
||||
(match) => {
|
||||
if (ALLOWED_URL_PATTERNS.some((pattern) => pattern.test(match))) {
|
||||
return match;
|
||||
}
|
||||
return "";
|
||||
},
|
||||
);
|
||||
|
||||
// Clean up double spaces from URL removal
|
||||
processed = processed.replace(/ +/gu, " ").trim();
|
||||
|
||||
// Strip markdown formatting (KakaoTalk is plaintext)
|
||||
processed = processed.replace(/\*\*(.+?)\*\*/gu, "$1"); // **bold**
|
||||
processed = processed.replace(/\*(.+?)\*/gu, "$1"); // *italic*
|
||||
processed = processed.replace(/__(.+?)__/gu, "$1"); // __underline__
|
||||
processed = processed.replace(/~~(.+?)~~/gu, "$1"); // ~~strike~~
|
||||
processed = processed.replace(/^#{1,6}\s+/gmu, ""); // ## headings
|
||||
processed = processed.replace(/^[-*]\s+/gmu, ""); // - or * list items
|
||||
processed = processed.replace(/`(.+?)`/gu, "$1"); // `inline code`
|
||||
|
||||
// Truncate to max length with Korean sentence boundary awareness
|
||||
const maxLen = getMaxResponseLength();
|
||||
if (processed.length > maxLen) {
|
||||
processed = processed.slice(0, maxLen).trimEnd();
|
||||
// Search backwards for a Korean sentence-ending pattern
|
||||
const sentenceEndPattern = /(?:세요[.!]?|해요[.!]?|돼요[.!]?|어요[.!]?|아요[.!]?|거든요[.!]?|습니다[.!]?|됩니다[.!]?|니다[.!]?|요[.!]|다[.!]|[.!?])\s*$/u;
|
||||
// Try progressively shorter substrings to find a sentence boundary
|
||||
for (let i = processed.length; i > maxLen * 0.5; i--) {
|
||||
const sub = processed.slice(0, i);
|
||||
if (sentenceEndPattern.test(sub)) {
|
||||
processed = sub;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return processed.trim();
|
||||
}
|
||||
|
||||
function isLowQualityResponse(
|
||||
text: string,
|
||||
options?: { allowShortResponse?: boolean },
|
||||
): boolean {
|
||||
if (!options?.allowShortResponse && text.length < 10) return true;
|
||||
// Check for excessive repetition (same phrase 3+ times)
|
||||
const words = text.split(/\s+/u);
|
||||
if (words.length >= 6) {
|
||||
const phrases = new Map<string, number>();
|
||||
for (let i = 0; i < words.length - 1; i++) {
|
||||
const bigram = `${words[i]} ${words[i + 1]}`;
|
||||
phrases.set(bigram, (phrases.get(bigram) ?? 0) + 1);
|
||||
}
|
||||
for (const count of phrases.values()) {
|
||||
if (count >= 3) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export class OnboardingSkill implements Skill {
|
||||
readonly name = "onboarding";
|
||||
private readonly classifier: OnboardingIntentClassifier;
|
||||
private readonly blackboardStore: OnboardingBlackboardStore;
|
||||
private readonly orchestrator: OnboardingOrchestrator;
|
||||
private readonly shadowEvalPublisher: ShadowEvalPublisher;
|
||||
private readonly tools: readonly Tool<unknown, unknown>[];
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
private readonly llmClient: LlmClient,
|
||||
private readonly roomContextService?: RoomContextService,
|
||||
shadowEvalPublisher?: ShadowEvalPublisher,
|
||||
) {
|
||||
this.classifier = new OnboardingIntentClassifier(llmClient);
|
||||
this.blackboardStore = new OnboardingBlackboardStore(garnetClient);
|
||||
this.orchestrator = new OnboardingOrchestrator(llmClient);
|
||||
this.shadowEvalPublisher =
|
||||
shadowEvalPublisher ?? new ShadowEvalBus(garnetClient);
|
||||
this.tools = createBuiltInToolRegistry().list();
|
||||
}
|
||||
|
||||
canHandle(request: AgentRequest): boolean {
|
||||
@@ -185,6 +128,15 @@ export class OnboardingSkill implements Skill {
|
||||
await this.rememberTurn(request, derived).catch((error) => {
|
||||
logger.warn({ err: error }, "[OnboardingSkill] Failed to update blackboard");
|
||||
});
|
||||
|
||||
this.shadowEvalPublisher.publish({
|
||||
classified,
|
||||
createdAt: new Date().toISOString(),
|
||||
derived,
|
||||
latencyMs: this.getCurrentLatencyMs(),
|
||||
request,
|
||||
response,
|
||||
});
|
||||
}
|
||||
|
||||
return response;
|
||||
@@ -212,11 +164,15 @@ export class OnboardingSkill implements Skill {
|
||||
derived,
|
||||
);
|
||||
|
||||
return this.generateLlmText(messages, {
|
||||
return this.generateLlmText(messages, derived, {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
maxTokens: derived.suggestedMaxTokens,
|
||||
retryOnLowQuality:
|
||||
request.trigger === "user-message" && !derived.allowShortResponse,
|
||||
request.trigger === "user-message" &&
|
||||
!derived.allowShortResponse &&
|
||||
derived.responseMode !== "llm-with-tools",
|
||||
toolContext: this.buildToolContext(request, session),
|
||||
tools: this.selectToolsForDerived(derived),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -456,11 +412,14 @@ export class OnboardingSkill implements Skill {
|
||||
|
||||
private async generateLlmText(
|
||||
messages: ChatMessage[],
|
||||
derived: DerivedState,
|
||||
options?: {
|
||||
allowShortResponse?: boolean;
|
||||
maxTokens?: number;
|
||||
retryOnLowQuality?: boolean;
|
||||
temperature?: number;
|
||||
toolContext?: ToolContext | undefined;
|
||||
tools?: readonly Tool<unknown, unknown>[] | undefined;
|
||||
},
|
||||
): Promise<LlmTextResult> {
|
||||
if (!this.llmClient.hasEndpoints()) {
|
||||
@@ -509,39 +468,79 @@ export class OnboardingSkill implements Skill {
|
||||
requestOptions.temperature = attempt.temperature;
|
||||
}
|
||||
|
||||
const llmResult = await this.llmClient.chatCompletion(
|
||||
const toolContext = options?.toolContext ?? {};
|
||||
const llmText = await this.orchestrator.run(
|
||||
derived,
|
||||
messages,
|
||||
requestOptions,
|
||||
{
|
||||
maxTokens: requestOptions.maxTokens,
|
||||
temperature: requestOptions.temperature,
|
||||
toolContext,
|
||||
tools: options?.tools,
|
||||
},
|
||||
);
|
||||
|
||||
if (!llmResult) {
|
||||
if (!llmText) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const processed = postProcessResponse(llmResult.content);
|
||||
if (!processed) {
|
||||
continue;
|
||||
}
|
||||
if (processed.includes("[NO_RESPONSE]")) {
|
||||
const validation = validateOnboardingResponse(
|
||||
llmText,
|
||||
derived,
|
||||
{
|
||||
allowShortResponse: options?.allowShortResponse,
|
||||
},
|
||||
);
|
||||
|
||||
if (validation.text.includes("[NO_RESPONSE]")) {
|
||||
return { status: "no_response" };
|
||||
}
|
||||
const qualityOptions =
|
||||
options?.allowShortResponse !== undefined
|
||||
? { allowShortResponse: options.allowShortResponse }
|
||||
: undefined;
|
||||
if (isLowQualityResponse(processed, qualityOptions)) {
|
||||
if (validation.status === "reject") {
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
status: "success",
|
||||
text: processed,
|
||||
text: validation.text,
|
||||
};
|
||||
}
|
||||
|
||||
return { status: "failed" };
|
||||
}
|
||||
|
||||
private buildToolContext(
|
||||
request: AgentRequest,
|
||||
session: OnboardingSession | null,
|
||||
): ToolContext {
|
||||
return {
|
||||
chatroomId: request.chatroomId,
|
||||
userId: request.userId,
|
||||
session,
|
||||
garnetClient: this.garnetClient,
|
||||
roomContextService: this.roomContextService,
|
||||
};
|
||||
}
|
||||
|
||||
private selectToolsForDerived(
|
||||
derived: DerivedState,
|
||||
): readonly Tool<unknown, unknown>[] {
|
||||
if (derived.phase.kind === "nickname-rejected") {
|
||||
return this.tools.filter((tool) =>
|
||||
tool.name === "validate-nickname" ||
|
||||
tool.name === "lookup-community-rule" ||
|
||||
tool.name === "check-onboarding-progress",
|
||||
);
|
||||
}
|
||||
|
||||
if (derived.intent.kind === "status-question") {
|
||||
return this.tools.filter((tool) =>
|
||||
tool.name === "check-onboarding-progress",
|
||||
);
|
||||
}
|
||||
|
||||
return this.tools;
|
||||
}
|
||||
|
||||
private makeTemplateResponse(
|
||||
requestId: string,
|
||||
text: string,
|
||||
@@ -574,4 +573,14 @@ export class OnboardingSkill implements Skill {
|
||||
source: "template",
|
||||
};
|
||||
}
|
||||
|
||||
private getCurrentLatencyMs(): number | undefined {
|
||||
const startedAt = getContext()?.startedAt;
|
||||
const startedTimestamp = startedAt ? Date.parse(startedAt) : NaN;
|
||||
if (!Number.isFinite(startedTimestamp)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return Math.max(0, Date.now() - startedTimestamp);
|
||||
}
|
||||
}
|
||||
@@ -1,16 +1,16 @@
|
||||
import type { Intent } from "./onboarding-state.js";
|
||||
|
||||
export const EXPECTED_KEYWORDS: Record<Intent["kind"], string[]> = {
|
||||
acknowledgement: ["감사", "알겠", "확인"],
|
||||
"age-question": ["27", "나이", "가입"],
|
||||
acknowledgement: ["네", "좋아요", "천천히"],
|
||||
"age-question": ["27", "가입", "가능"],
|
||||
chitchat: [],
|
||||
"discord-question": ["디스코드", "디코", "discord"],
|
||||
greeting: ["안녕", "반갑", "환영"],
|
||||
"help-how": ["어떻게", "방법", "해"],
|
||||
"help-where": ["어디", "위치", "찾"],
|
||||
"nickname-question": ["닉네임", "닉변", "형식", "이름"],
|
||||
"discord-question": ["디스코드", "권한", "완료"],
|
||||
greeting: ["반갑", "환영", "어서 오세요"],
|
||||
"help-how": ["하트", "닉네임", "스크린샷", "형식", "커버 보기"],
|
||||
"help-where": ["오른쪽 상단", "커버 보기", "프로필"],
|
||||
"nickname-question": ["닉네임", "형식", "티어", "/"],
|
||||
"off-topic-to-other": [],
|
||||
"photo-question": ["하트", "인증", "스크린샷", "프로필"],
|
||||
"status-question": ["남았", "완료", "진행", "상태"],
|
||||
"photo-question": ["하트", "스크린샷", "프로필", "커버 보기"],
|
||||
"status-question": ["남았", "완료", "아직", "끝"],
|
||||
unknown: [],
|
||||
};
|
||||
@@ -1,4 +1,4 @@
|
||||
import { getDeadlineNearThresholdMin } from "../../../config.js";
|
||||
import { DEADLINE_NEAR_THRESHOLD_MIN } from "../constants.js";
|
||||
import { analyzeNicknameIssues } from "../onboarding-templates.js";
|
||||
import type { OnboardingSession } from "../types.js";
|
||||
import type { OnboardingPhase, RemainingStep } from "./onboarding-state.js";
|
||||
@@ -30,7 +30,7 @@ function isDeadlineNear(session: OnboardingSession | null): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
return deadlineAt - Date.now() <= getDeadlineNearThresholdMin() * 60_000;
|
||||
return deadlineAt - Date.now() <= DEADLINE_NEAR_THRESHOLD_MIN * 60_000;
|
||||
}
|
||||
|
||||
export function derivePhaseFromSession(
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { z } from "zod";
|
||||
import { getOnboardingSessionKeyPrefix } from "../../config.js";
|
||||
import type { OnboardingSession } from "@skills/onboarding/types";
|
||||
import type { Tool } from "../tool.js";
|
||||
|
||||
const argsSchema = z.object({
|
||||
chatroomId: z.string().min(1),
|
||||
userId: z.string().min(1),
|
||||
});
|
||||
|
||||
const resultSchema = z.object({
|
||||
nicknameDone: z.boolean(),
|
||||
photoDone: z.boolean(),
|
||||
remainingSteps: z.array(z.enum(["nickname", "photo"])),
|
||||
deadlineInHours: z.number().nullable(),
|
||||
});
|
||||
|
||||
function getRemainingSteps(session: OnboardingSession | null): Array<"nickname" | "photo"> {
|
||||
if (!session) {
|
||||
return ["nickname", "photo"];
|
||||
}
|
||||
|
||||
const remaining: Array<"nickname" | "photo"> = [];
|
||||
if (!session.nicknameVerified) {
|
||||
remaining.push("nickname");
|
||||
}
|
||||
if (!session.photoVerified) {
|
||||
remaining.push("photo");
|
||||
}
|
||||
return remaining;
|
||||
}
|
||||
|
||||
export const checkOnboardingProgressTool: Tool<
|
||||
z.infer<typeof argsSchema>,
|
||||
z.infer<typeof resultSchema>
|
||||
> = {
|
||||
name: "check-onboarding-progress",
|
||||
description:
|
||||
"현재 유저의 온보딩 진행 상태를 읽어 남은 절차와 데드라인까지 남은 시간을 구조화해서 반환한다.",
|
||||
argsSchema,
|
||||
resultSchema,
|
||||
async execute(args, ctx) {
|
||||
const session =
|
||||
ctx.session && ctx.chatroomId === args.chatroomId && ctx.userId === args.userId
|
||||
? ctx.session
|
||||
: ctx.garnetClient
|
||||
? await ctx.garnetClient.getJson<OnboardingSession>(
|
||||
`${getOnboardingSessionKeyPrefix()}:onboarding:${args.chatroomId}:${args.userId}`,
|
||||
)
|
||||
: null;
|
||||
|
||||
const deadlineAt = session?.deadlineAt ? Date.parse(session.deadlineAt) : NaN;
|
||||
const deadlineInHours = Number.isFinite(deadlineAt)
|
||||
? Math.max(0, (deadlineAt - Date.now()) / 3_600_000)
|
||||
: null;
|
||||
|
||||
return {
|
||||
nicknameDone: session?.nicknameVerified ?? false,
|
||||
photoDone: session?.photoVerified ?? false,
|
||||
remainingSteps: getRemainingSteps(session),
|
||||
deadlineInHours,
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
import { z } from "zod";
|
||||
import type { Tool } from "../tool.js";
|
||||
|
||||
const argsSchema = z.object({
|
||||
chatroomId: z.string().min(1),
|
||||
limit: z.number().int().min(1).max(20).optional(),
|
||||
});
|
||||
|
||||
const roomMessageSchema = z.object({
|
||||
senderId: z.string(),
|
||||
nickname: z.string(),
|
||||
message: z.string(),
|
||||
timestamp: z.string(),
|
||||
});
|
||||
|
||||
const resultSchema = z.object({
|
||||
messages: z.array(roomMessageSchema),
|
||||
});
|
||||
|
||||
export const getRoomContextTool: Tool<
|
||||
z.infer<typeof argsSchema>,
|
||||
z.infer<typeof resultSchema>
|
||||
> = {
|
||||
name: "get-room-context",
|
||||
description:
|
||||
"최근 채팅방 대화를 조회해서 현재 유저 메시지가 다른 사람에게 향한 말인지, 방 분위기가 어떤지 판단할 수 있게 돕는다.",
|
||||
argsSchema,
|
||||
resultSchema,
|
||||
async execute(args, ctx) {
|
||||
if (!ctx.roomContextService) {
|
||||
return { messages: [] };
|
||||
}
|
||||
|
||||
const messages = await ctx.roomContextService.getRecentMessages(
|
||||
args.chatroomId,
|
||||
args.limit,
|
||||
);
|
||||
|
||||
return {
|
||||
messages: messages.map((message) => ({
|
||||
senderId: message.senderId,
|
||||
nickname: message.nickname,
|
||||
message: message.message,
|
||||
timestamp: message.timestamp.toISOString(),
|
||||
})),
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,20 @@
|
||||
import { ToolRegistry } from "../registry.js";
|
||||
import { checkOnboardingProgressTool } from "./check-onboarding-progress.js";
|
||||
import { getRoomContextTool } from "./get-room-context.js";
|
||||
import { lookupCommunityRuleTool } from "./lookup-community-rule.js";
|
||||
import { validateNicknameTool } from "./validate-nickname.js";
|
||||
|
||||
export function createBuiltInToolRegistry(): ToolRegistry {
|
||||
return new ToolRegistry()
|
||||
.register(validateNicknameTool)
|
||||
.register(checkOnboardingProgressTool)
|
||||
.register(lookupCommunityRuleTool)
|
||||
.register(getRoomContextTool);
|
||||
}
|
||||
|
||||
export {
|
||||
checkOnboardingProgressTool,
|
||||
getRoomContextTool,
|
||||
lookupCommunityRuleTool,
|
||||
validateNicknameTool,
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
import { z } from "zod";
|
||||
import {
|
||||
DEADLINE_NEAR_THRESHOLD_MIN,
|
||||
DISCORD_GUIDE_TEXT,
|
||||
HEART_RULE_TEXT,
|
||||
MIN_AGE,
|
||||
NICKNAME_FORMAT_TEXT,
|
||||
TIER_CODES_TEXT,
|
||||
} from "@skills/onboarding/constants";
|
||||
import type { Tool } from "../tool.js";
|
||||
|
||||
const argsSchema = z.object({
|
||||
topic: z.enum([
|
||||
"age",
|
||||
"tier-codes",
|
||||
"discord-link",
|
||||
"heart-rule",
|
||||
"nickname-format",
|
||||
"deadline",
|
||||
]),
|
||||
});
|
||||
|
||||
const resultSchema = z.object({
|
||||
topic: argsSchema.shape.topic,
|
||||
text: z.string(),
|
||||
source: z.literal("constants"),
|
||||
});
|
||||
|
||||
const TOPIC_TEXT: Record<z.infer<typeof argsSchema>["topic"], string> = {
|
||||
age: `가입 가능 연령은 ${MIN_AGE}세 이상이에요.`,
|
||||
"tier-codes": `사용 가능한 티어 코드는 ${TIER_CODES_TEXT}예요.`,
|
||||
"discord-link": DISCORD_GUIDE_TEXT,
|
||||
"heart-rule": HEART_RULE_TEXT,
|
||||
"nickname-format": NICKNAME_FORMAT_TEXT,
|
||||
deadline: `데드라인이 ${DEADLINE_NEAR_THRESHOLD_MIN}분 이하로 남으면 마감이 임박한 상태로 안내해요.`,
|
||||
};
|
||||
|
||||
export const lookupCommunityRuleTool: Tool<
|
||||
z.infer<typeof argsSchema>,
|
||||
z.infer<typeof resultSchema>
|
||||
> = {
|
||||
name: "lookup-community-rule",
|
||||
description:
|
||||
"나이 제한, 닉네임 형식, 티어 코드, 하트 인증 규칙, 디스코드 안내 같은 커뮤니티 규칙을 상수에서 조회한다.",
|
||||
argsSchema,
|
||||
resultSchema,
|
||||
execute(args) {
|
||||
return Promise.resolve({
|
||||
topic: args.topic,
|
||||
text: TOPIC_TEXT[args.topic],
|
||||
source: "constants",
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,52 @@
|
||||
import { z } from "zod";
|
||||
import { analyzeNicknameIssues } from "@skills/onboarding/onboarding-templates";
|
||||
import type { Tool } from "../tool.js";
|
||||
|
||||
const argsSchema = z.object({
|
||||
nickname: z.string().min(1),
|
||||
});
|
||||
|
||||
const nicknameIssueSchema = z.object({
|
||||
kind: z.enum([
|
||||
"missing-tier",
|
||||
"invalid-age",
|
||||
"missing-gender",
|
||||
"wrong-separator",
|
||||
"unknown-tier",
|
||||
"other",
|
||||
]),
|
||||
message: z.string(),
|
||||
suggestion: z.string().optional(),
|
||||
});
|
||||
|
||||
const resultSchema = z.object({
|
||||
valid: z.boolean(),
|
||||
issues: z.array(nicknameIssueSchema),
|
||||
normalized: z.string().optional(),
|
||||
});
|
||||
|
||||
export const validateNicknameTool: Tool<
|
||||
z.infer<typeof argsSchema>,
|
||||
z.infer<typeof resultSchema>
|
||||
> = {
|
||||
name: "validate-nickname",
|
||||
description:
|
||||
"새내기 닉네임 형식이 이름/출생 연도/성별/티어 규칙에 맞는지 검사하고, 틀린 부분을 구조화해서 반환한다.",
|
||||
argsSchema,
|
||||
resultSchema,
|
||||
execute(args) {
|
||||
const normalized = args.nickname.trim();
|
||||
const issues = analyzeNicknameIssues(normalized);
|
||||
|
||||
const result: z.infer<typeof resultSchema> = {
|
||||
valid: issues.length === 0,
|
||||
issues,
|
||||
};
|
||||
|
||||
if (issues.length === 0) {
|
||||
result.normalized = normalized;
|
||||
}
|
||||
|
||||
return Promise.resolve(result);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,8 @@
|
||||
export { ToolRegistry } from "./registry.js";
|
||||
export { parseToolCalls } from "./parser.js";
|
||||
export {
|
||||
serializeToolsForHermes,
|
||||
serializeToolsForOpenAi,
|
||||
} from "./serializer.js";
|
||||
export type { OpenAiToolDefinition } from "./serializer.js";
|
||||
export type { Tool, ToolContext } from "./tool.js";
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { parseToolCalls } from "./parser.js";
|
||||
|
||||
describe("parseToolCalls", () => {
|
||||
it("parses Hermes tool call blocks", () => {
|
||||
const result = parseToolCalls(
|
||||
'<tool_call>{"name":"validate-nickname","arguments":{"nickname":"프리리/95/여/D"}}</tool_call>',
|
||||
);
|
||||
|
||||
expect(result).toEqual([
|
||||
{
|
||||
name: "validate-nickname",
|
||||
arguments: {
|
||||
nickname: "프리리/95/여/D",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("parses OpenAI tool_calls arrays", () => {
|
||||
const result = parseToolCalls({
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
function: {
|
||||
name: "lookup-community-rule",
|
||||
arguments: '{"topic":"age"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual([
|
||||
{
|
||||
id: "call_1",
|
||||
name: "lookup-community-rule",
|
||||
arguments: {
|
||||
topic: "age",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("parses tool calls nested in choices[0].message", () => {
|
||||
const result = parseToolCalls({
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_2",
|
||||
function: {
|
||||
name: "check-onboarding-progress",
|
||||
arguments: '{"chatroomId":"room-1","userId":"user-1"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0]?.name).toBe("check-onboarding-progress");
|
||||
expect(result[0]?.arguments).toEqual({
|
||||
chatroomId: "room-1",
|
||||
userId: "user-1",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
export interface ParsedToolCall {
|
||||
id?: string | undefined;
|
||||
name: string;
|
||||
arguments: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface OpenAiToolCallShape {
|
||||
id?: string;
|
||||
function?: {
|
||||
name?: string;
|
||||
arguments?: string;
|
||||
};
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function parseJsonArguments(raw: unknown): Record<string, unknown> | null {
|
||||
if (typeof raw === "string") {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return asRecord(parsed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof raw === "object" && raw !== null) {
|
||||
return raw as Record<string, unknown>;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseHermesCalls(content: string): ParsedToolCall[] {
|
||||
const matches = [...content.matchAll(/<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/gu)];
|
||||
|
||||
return matches.flatMap((match) => {
|
||||
const payload = match[1];
|
||||
if (!payload) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(payload) as
|
||||
| {
|
||||
id?: string;
|
||||
name?: string;
|
||||
arguments?: unknown;
|
||||
}
|
||||
| Array<{
|
||||
id?: string;
|
||||
name?: string;
|
||||
arguments?: unknown;
|
||||
}>;
|
||||
|
||||
const items = Array.isArray(parsed) ? parsed : [parsed];
|
||||
return items.flatMap((item) => {
|
||||
const args = parseJsonArguments(item.arguments);
|
||||
if (!item.name || !args) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
arguments: args,
|
||||
},
|
||||
];
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function parseOpenAiCalls(value: unknown): ParsedToolCall[] {
|
||||
const record = asRecord(value);
|
||||
const choices = Array.isArray(record["choices"])
|
||||
? (record["choices"] as unknown[])
|
||||
: [];
|
||||
const firstChoice = choices[0];
|
||||
const candidates = [
|
||||
record["tool_calls"],
|
||||
asRecord(record["message"])["tool_calls"],
|
||||
asRecord(asRecord(firstChoice)["message"])["tool_calls"],
|
||||
];
|
||||
|
||||
const toolCallsCandidate = candidates.find(Array.isArray);
|
||||
if (!Array.isArray(toolCallsCandidate)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return toolCallsCandidate.flatMap((item) => {
|
||||
const call = item as OpenAiToolCallShape;
|
||||
const args = parseJsonArguments(call.function?.arguments);
|
||||
const name = call.function?.name;
|
||||
|
||||
if (!name || !args) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: call.id,
|
||||
name,
|
||||
arguments: args,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export function parseToolCalls(value: unknown): ParsedToolCall[] {
|
||||
if (typeof value === "string") {
|
||||
return parseHermesCalls(value);
|
||||
}
|
||||
|
||||
const record = asRecord(value);
|
||||
const content =
|
||||
typeof record["content"] === "string"
|
||||
? record["content"]
|
||||
: typeof asRecord(record["message"])["content"] === "string"
|
||||
? (asRecord(record["message"])["content"] as string)
|
||||
: "";
|
||||
|
||||
const parsedOpenAiCalls = parseOpenAiCalls(value);
|
||||
if (parsedOpenAiCalls.length > 0) {
|
||||
return parsedOpenAiCalls;
|
||||
}
|
||||
|
||||
return content ? parseHermesCalls(content) : [];
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { Tool } from "./tool.js";
|
||||
|
||||
export class ToolRegistry {
|
||||
private readonly tools = new Map<string, Tool<unknown, unknown>>();
|
||||
|
||||
register<A, R>(tool: Tool<A, R>): this {
|
||||
this.tools.set(tool.name, tool as Tool<unknown, unknown>);
|
||||
return this;
|
||||
}
|
||||
|
||||
get(name: string): Tool<unknown, unknown> {
|
||||
const tool = this.tools.get(name);
|
||||
if (!tool) {
|
||||
throw new Error(`Unknown tool: ${name}`);
|
||||
}
|
||||
return tool;
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this.tools.has(name);
|
||||
}
|
||||
|
||||
list(): Tool<unknown, unknown>[] {
|
||||
return [...this.tools.values()];
|
||||
}
|
||||
|
||||
pick(names: readonly string[]): Tool<unknown, unknown>[] {
|
||||
return names
|
||||
.filter((name) => this.tools.has(name))
|
||||
.map((name) => this.get(name));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { z } from "zod";
|
||||
import type { Tool } from "./tool.js";
|
||||
|
||||
export interface OpenAiToolDefinition {
|
||||
type: "function";
|
||||
function: {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
function toToolDescriptor(tool: Tool<unknown, unknown>): {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: unknown;
|
||||
} {
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: z.toJSONSchema(tool.argsSchema),
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeToolsForHermes(
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
): string {
|
||||
const descriptors = tools.map(toToolDescriptor);
|
||||
return `<tools>\n${JSON.stringify(descriptors, null, 2)}\n</tools>`;
|
||||
}
|
||||
|
||||
export function serializeToolsForOpenAi(
|
||||
tools: readonly Tool<unknown, unknown>[],
|
||||
): OpenAiToolDefinition[] {
|
||||
return tools.map((tool) => {
|
||||
const descriptor = toToolDescriptor(tool);
|
||||
return {
|
||||
type: "function",
|
||||
function: descriptor,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { GarnetClient } from "@common/garnet";
|
||||
import type { RoomContextService } from "@room/room-context-service";
|
||||
import type { ZodType } from "zod";
|
||||
import type { OnboardingSession } from "@skills/onboarding/types";
|
||||
|
||||
export interface ToolContext {
|
||||
chatroomId?: string | undefined;
|
||||
userId?: string | undefined;
|
||||
session?: OnboardingSession | null | undefined;
|
||||
garnetClient?: GarnetClient | undefined;
|
||||
roomContextService?: RoomContextService | undefined;
|
||||
}
|
||||
|
||||
export interface Tool<A, R> {
|
||||
name: string;
|
||||
description: string;
|
||||
argsSchema: ZodType<A>;
|
||||
resultSchema: ZodType<R>;
|
||||
execute(args: A, ctx: ToolContext): Promise<R>;
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import { loadEnvironmentFromNodeEnv } from "../common/env-loader.js";
|
||||
import {
|
||||
getOpenRouterApiUrl,
|
||||
getOpenRouterEnabled,
|
||||
} from "../config.js";
|
||||
|
||||
loadEnvironmentFromNodeEnv();
|
||||
|
||||
const baseServiceName = process.env["SERVICE_NAME"]?.trim() || "lolympus-agent";
|
||||
process.env["SERVICE_NAME"] = `${baseServiceName}-worker`;
|
||||
|
||||
const { getLogger, initLogger } = await import("../observability/logger.js");
|
||||
initLogger();
|
||||
const logger = getLogger();
|
||||
const { initTracing, shutdownTracing } = await import(
|
||||
"../observability/tracing.js"
|
||||
);
|
||||
initTracing();
|
||||
|
||||
const { GarnetClient } = await import("../common/garnet.js");
|
||||
const { NatsClient } = await import("../common/nats.js");
|
||||
const { LlmClient } = await import("../llm/index.js");
|
||||
const { getRawSecret } = await import("../common/secret-manager.js");
|
||||
const { ConversationStore } = await import("../conversation/index.js");
|
||||
const { ShadowWorker } = await import("../eval/shadow-worker.js");
|
||||
const { AlertService } = await import("../observability/alert.js");
|
||||
|
||||
function firstNonEmpty(
|
||||
...values: Array<string | null | undefined>
|
||||
): string | undefined {
|
||||
for (const value of values) {
|
||||
if (value?.trim()) {
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseEndpointList(
|
||||
...values: Array<string | null | undefined>
|
||||
): string[] {
|
||||
const items: string[] = [];
|
||||
|
||||
for (const value of values) {
|
||||
if (!value?.trim()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const item of value.split(/[\n,]/u)) {
|
||||
const trimmed = item.trim();
|
||||
if (trimmed) {
|
||||
items.push(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...new Set(items)];
|
||||
}
|
||||
|
||||
function toProviderEndpoints(
|
||||
urls: string[],
|
||||
provider?: "llama" | "cliproxy" | "openrouter",
|
||||
): Array<string | { url: string; provider?: "llama" | "cliproxy" | "openrouter" }> {
|
||||
if (!provider) {
|
||||
return urls;
|
||||
}
|
||||
|
||||
return urls.map((url) => ({ url, provider }));
|
||||
}
|
||||
|
||||
function installShutdownHooks(
|
||||
shutdown: (reason: string, error?: unknown) => Promise<void>,
|
||||
): void {
|
||||
let shuttingDown = false;
|
||||
|
||||
const runShutdown = async (
|
||||
reason: string,
|
||||
error?: unknown,
|
||||
): Promise<void> => {
|
||||
if (shuttingDown) {
|
||||
return;
|
||||
}
|
||||
shuttingDown = true;
|
||||
|
||||
if (error) {
|
||||
logger.error({ err: error }, `[Background] ${reason}`);
|
||||
} else {
|
||||
logger.info(`[Background] ${reason}`);
|
||||
}
|
||||
|
||||
try {
|
||||
await shutdown(reason, error);
|
||||
} catch (shutdownError) {
|
||||
logger.error({ err: shutdownError }, "[Background] Cleanup failed");
|
||||
} finally {
|
||||
process.exit(error ? 1 : 0);
|
||||
}
|
||||
};
|
||||
|
||||
for (const signal of ["SIGINT", "SIGTERM"] as const) {
|
||||
process.once(signal, () => {
|
||||
void runShutdown(signal);
|
||||
});
|
||||
}
|
||||
|
||||
process.once("uncaughtException", (error) => {
|
||||
void runShutdown("uncaughtException", error);
|
||||
});
|
||||
|
||||
process.once("unhandledRejection", (reason) => {
|
||||
void runShutdown("unhandledRejection", reason);
|
||||
});
|
||||
}
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const serviceName = process.env["SERVICE_NAME"] ?? "lolympus-agent-worker";
|
||||
|
||||
const garnetClient = new GarnetClient();
|
||||
await garnetClient.connect();
|
||||
const natsClient = new NatsClient();
|
||||
await natsClient.connect();
|
||||
|
||||
const cliproxyEndpoints = parseEndpointList(
|
||||
getRawSecret("CLI_PROXY_API_URLS"),
|
||||
process.env["CLI_PROXY_API_URLS"],
|
||||
);
|
||||
const generalLlmEndpoints = parseEndpointList(
|
||||
getRawSecret("AGENT_LLM_API_URLS"),
|
||||
process.env["AGENT_LLM_API_URLS"],
|
||||
getRawSecret("LLAMA_SERVER_API_URL"),
|
||||
process.env["LLAMA_SERVER_API_URL"],
|
||||
);
|
||||
const openRouterEndpoints = getOpenRouterEnabled()
|
||||
? parseEndpointList(
|
||||
getRawSecret("OPENROUTER_API_URL"),
|
||||
process.env["OPENROUTER_API_URL"],
|
||||
getOpenRouterApiUrl(),
|
||||
)
|
||||
: [];
|
||||
const llmApiKey = firstNonEmpty(
|
||||
getRawSecret("AGENT_LLM_API_KEY"),
|
||||
process.env["AGENT_LLM_API_KEY"],
|
||||
getRawSecret("LLAMA_SERVER_API_KEY"),
|
||||
process.env["LLAMA_SERVER_API_KEY"],
|
||||
);
|
||||
const providerApiKeys = Object.fromEntries(
|
||||
[
|
||||
[
|
||||
"cliproxy",
|
||||
firstNonEmpty(
|
||||
getRawSecret("CLI_PROXY_API_KEY"),
|
||||
process.env["CLI_PROXY_API_KEY"],
|
||||
),
|
||||
],
|
||||
["llama", llmApiKey],
|
||||
[
|
||||
"openrouter",
|
||||
firstNonEmpty(
|
||||
getRawSecret("OPENROUTER_API_KEY"),
|
||||
process.env["OPENROUTER_API_KEY"],
|
||||
),
|
||||
],
|
||||
].filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
||||
) as Partial<Record<"cliproxy" | "llama" | "openrouter", string>>;
|
||||
|
||||
const llmClient = new LlmClient({
|
||||
endpoints: [
|
||||
...toProviderEndpoints(cliproxyEndpoints, "cliproxy"),
|
||||
...toProviderEndpoints(openRouterEndpoints, "openrouter"),
|
||||
...toProviderEndpoints(generalLlmEndpoints, "llama"),
|
||||
],
|
||||
garnetClient,
|
||||
apiKey: llmApiKey,
|
||||
model: firstNonEmpty(
|
||||
getRawSecret("AGENT_LLM_MODEL"),
|
||||
process.env["AGENT_LLM_MODEL"],
|
||||
),
|
||||
providerApiKeys,
|
||||
});
|
||||
const conversationStore = new ConversationStore(garnetClient);
|
||||
const alertService = new AlertService(garnetClient, natsClient);
|
||||
const worker = new ShadowWorker(
|
||||
garnetClient,
|
||||
llmClient,
|
||||
conversationStore,
|
||||
alertService,
|
||||
);
|
||||
|
||||
installShutdownHooks(async () => {
|
||||
await worker.stop();
|
||||
await natsClient.close();
|
||||
await garnetClient.close();
|
||||
await shutdownTracing();
|
||||
});
|
||||
|
||||
await worker.start();
|
||||
logger.info({ serviceName }, "✓ Background worker bootstrapped");
|
||||
}
|
||||
|
||||
try {
|
||||
await bootstrap();
|
||||
} catch (error) {
|
||||
logger.error({ err: error }, "[Background] Fatal startup error");
|
||||
await shutdownTracing();
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Reference in new issue
Block a user