1262 lines
34 KiB
TypeScript
1262 lines
34 KiB
TypeScript
import { existsSync } from "node:fs";
|
|
import { readFile } from "node:fs/promises";
|
|
import { resolve } from "node:path";
|
|
|
|
import type { AgentRequest, AgentResponse } from "@agent/types";
|
|
import type {
|
|
GarnetClient,
|
|
StreamEntry,
|
|
StreamPendingEntry,
|
|
} from "@common/garnet";
|
|
import { summarizeConversation } from "@conv/summarizer";
|
|
import type { LlmClient } from "@llm/llm-client";
|
|
import { getLogger } from "@obs/logger";
|
|
import type { AlertSender } from "@obs/alert";
|
|
import type { ConversationStore } from "@conv/conversation-store";
|
|
import {
|
|
getAlertThresholdDriftPct,
|
|
getAlertThresholdEvalScoreDropPct,
|
|
getAlertThresholdLlmErrorRate,
|
|
getAlertThresholdP95LatencyMs,
|
|
getEvalAggTickMs,
|
|
getEvalArchiveTickMs,
|
|
getEvalArchiveBatchSize,
|
|
getEvalAttributionTickMs,
|
|
getEvalDriftTickMs,
|
|
getEvalProcessLlmJudgeEnabled,
|
|
getEvalProcessLlmJudgeSamplingRate,
|
|
getEvalPrmEnabled,
|
|
getEvalPrmWeightsPath,
|
|
getEvalPromoteTickMs,
|
|
getEvalReviewTickMs,
|
|
getEvalStreamMaxLen,
|
|
} from "../config.js";
|
|
import {
|
|
aggregateRecentEvalResults,
|
|
EVAL_AGG_CURSOR_KEY,
|
|
sliceKeyToString,
|
|
} from "./aggregator.js";
|
|
import {
|
|
archiveEvalResults,
|
|
EVAL_ARCHIVE_CURSOR_KEY,
|
|
getEvalArchiveCollection,
|
|
type MongoConnectionLike,
|
|
} from "./archive.js";
|
|
import { computeDriftReports } from "./drift.js";
|
|
import { judgeSkillWithLlm } from "./judges/llm-judge.js";
|
|
import { writeProcessAttributions } from "./judges/attribution.js";
|
|
import { loadPrmModel } from "./judges/prm.js";
|
|
import {
|
|
judgeProcessRule,
|
|
judgeProcessWithLlm,
|
|
} from "./judges/process-judge.js";
|
|
import { getSkillJudgeConfig } from "./judges/skill-config.js";
|
|
import {
|
|
appendCandidatesToJsonl,
|
|
selectPromotionCandidates,
|
|
} from "./promote.js";
|
|
import {
|
|
EVAL_REVIEW_CURSOR_KEY,
|
|
routeReviewCandidates,
|
|
} from "./review/queue.js";
|
|
import {
|
|
archiveReviewLabels,
|
|
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
|
} from "./review/label-archive.js";
|
|
import type {
|
|
EvalBaselineFile,
|
|
EvalCaseDefinition,
|
|
EvalDerivedSnapshot,
|
|
EvalProcessTraceSnapshot,
|
|
ProcessJudgeResult,
|
|
} from "./types.js";
|
|
|
|
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
|
const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
|
|
const BACKGROUND_GROUP = "bg-worker";
|
|
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
|
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
|
const EVAL_BASELINE_FILES = [
|
|
"eval-reports/baseline.onboarding.json",
|
|
"eval-reports/baseline.json",
|
|
];
|
|
const ALERT_WINDOW_MS = 5 * 60_000;
|
|
const ALERT_WINDOW_LABEL = "5m";
|
|
const ALERT_TICK_MS = 60_000;
|
|
const MAX_DELIVERIES = 3;
|
|
const MIN_ERROR_RATE_OBSERVATIONS = 5;
|
|
const MIN_LATENCY_OBSERVATIONS = 5;
|
|
const MIN_SCORE_OBSERVATIONS = 3;
|
|
const PENDING_IDLE_MS = 1_000;
|
|
const READ_BLOCK_MS = 5_000;
|
|
const READ_COUNT = 10;
|
|
const PENDING_BATCH_SIZE = 20;
|
|
const STOP_TIMEOUT_MS = 10_000;
|
|
|
|
type BackgroundJobType = "shadow-eval" | "summarize";
|
|
|
|
interface ShadowEvalJob {
|
|
classified: unknown;
|
|
createdAt: string;
|
|
derived: unknown;
|
|
latencyMs?: number | undefined;
|
|
process?: EvalProcessTraceSnapshot | undefined;
|
|
request: AgentRequest;
|
|
response: AgentResponse;
|
|
skill: string;
|
|
version: number;
|
|
}
|
|
|
|
interface EvalObservation {
|
|
llmJudgeErrored: boolean;
|
|
llmOverall: number | null;
|
|
recordedAt: number;
|
|
responseLatencyMs: number | null;
|
|
skill: string;
|
|
}
|
|
|
|
function delay(ms: number): Promise<void> {
|
|
return new Promise((resolve) => {
|
|
setTimeout(resolve, ms);
|
|
});
|
|
}
|
|
|
|
function promiseWithTimeout<T>(
|
|
promise: Promise<T>,
|
|
timeoutMs: number,
|
|
): Promise<T> {
|
|
return Promise.race([
|
|
promise,
|
|
new Promise<T>((_, reject) => {
|
|
setTimeout(() => {
|
|
reject(new Error(`Timed out after ${timeoutMs}ms`));
|
|
}, timeoutMs);
|
|
}),
|
|
]);
|
|
}
|
|
|
|
function getJobType(message: StreamEntry): string {
|
|
return message.message["type"] ?? "unknown";
|
|
}
|
|
|
|
function parseOptionalNumberField(value: string | undefined): number | undefined {
|
|
if (!value?.trim()) {
|
|
return undefined;
|
|
}
|
|
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
}
|
|
|
|
function parseJsonField<T>(
|
|
value: string | undefined,
|
|
fieldName: string,
|
|
): T {
|
|
if (!value?.trim()) {
|
|
throw new Error(`Missing ${fieldName} field`);
|
|
}
|
|
|
|
return JSON.parse(value) as T;
|
|
}
|
|
|
|
function parseOptionalJsonField<T>(value: string | undefined): T | undefined {
|
|
if (!value?.trim()) {
|
|
return undefined;
|
|
}
|
|
|
|
return JSON.parse(value) as T;
|
|
}
|
|
|
|
function asRecord(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function getStringField(
|
|
record: Record<string, unknown>,
|
|
key: string,
|
|
): string | null {
|
|
const value = record[key];
|
|
return typeof value === "string" && value.trim() ? value : null;
|
|
}
|
|
|
|
function getNestedString(
|
|
record: Record<string, unknown>,
|
|
first: string,
|
|
second: string,
|
|
): string | null {
|
|
return getStringField(asRecord(record[first]), second);
|
|
}
|
|
|
|
function parseTier(value: unknown): 0 | 1 | 2 {
|
|
const parsed = typeof value === "number" ? value : Number(value);
|
|
return parsed === 0 || parsed === 1 || parsed === 2 ? parsed : 0;
|
|
}
|
|
|
|
function parseResponseMode(value: unknown): EvalDerivedSnapshot["responseMode"] {
|
|
return value === "skip" ||
|
|
value === "template" ||
|
|
value === "llm" ||
|
|
value === "llm-with-tools"
|
|
? value
|
|
: "llm";
|
|
}
|
|
|
|
function toDerivedSnapshot(derived: unknown): EvalDerivedSnapshot {
|
|
const record = asRecord(derived);
|
|
const intent = getStringField(record, "intent") ??
|
|
getNestedString(record, "intent", "kind") ??
|
|
"unknown";
|
|
const phase = getStringField(record, "phase") ??
|
|
getNestedString(record, "phase", "kind") ??
|
|
"unknown";
|
|
const abstentionScore = record["abstentionScore"];
|
|
const suggestedMaxTokens = record["suggestedMaxTokens"];
|
|
|
|
return {
|
|
...(typeof abstentionScore === "number"
|
|
? { abstentionScore }
|
|
: {}),
|
|
allowShortResponse: record["allowShortResponse"] === true,
|
|
intent,
|
|
phase,
|
|
responseMode: parseResponseMode(record["responseMode"]),
|
|
...(record["routingDecision"]
|
|
? { routingDecision: record["routingDecision"] as EvalDerivedSnapshot["routingDecision"] }
|
|
: {}),
|
|
shouldRespond: record["shouldRespond"] !== false,
|
|
suggestedMaxTokens:
|
|
typeof suggestedMaxTokens === "number" && Number.isFinite(suggestedMaxTokens)
|
|
? suggestedMaxTokens
|
|
: 192,
|
|
tier: parseTier(record["tier"]),
|
|
};
|
|
}
|
|
|
|
function getTraceAbstention(process: EvalProcessTraceSnapshot | undefined):
|
|
| NonNullable<EvalProcessTraceSnapshot["steps"]["classify"]>["output"]["abstention"]
|
|
| undefined {
|
|
return process?.steps.classify?.output.abstention;
|
|
}
|
|
|
|
function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition {
|
|
return {
|
|
id: `shadow:${messageId}`,
|
|
lineNumber: 0,
|
|
input: {
|
|
request: job.request,
|
|
},
|
|
expected: {},
|
|
};
|
|
}
|
|
|
|
function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
|
const input = parseJsonField<{ request: AgentRequest }>(
|
|
message.message["input"],
|
|
"input",
|
|
);
|
|
const process = parseOptionalJsonField<EvalProcessTraceSnapshot>(
|
|
message.message["process"],
|
|
);
|
|
|
|
const job: ShadowEvalJob = {
|
|
classified: parseJsonField<unknown>(
|
|
message.message["classified"],
|
|
"classified",
|
|
),
|
|
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
|
derived: parseJsonField<unknown>(
|
|
message.message["derived"],
|
|
"derived",
|
|
),
|
|
latencyMs: parseOptionalNumberField(message.message["latencyMs"]),
|
|
request: input.request,
|
|
response: parseJsonField<AgentResponse>(
|
|
message.message["output"],
|
|
"output",
|
|
),
|
|
skill: message.message["skill"]?.trim() || input.request.skill || "onboarding",
|
|
version: Number(message.message["version"] ?? 1),
|
|
};
|
|
|
|
if (process) {
|
|
job.process = process;
|
|
}
|
|
|
|
return job;
|
|
}
|
|
|
|
export function calculatePercentile(
|
|
values: number[],
|
|
percentile: number,
|
|
): number | null {
|
|
if (values.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
const sorted = [...values].sort((left, right) => left - right);
|
|
const index = Math.min(
|
|
sorted.length - 1,
|
|
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
|
);
|
|
const selected = sorted[index];
|
|
return selected === undefined ? null : selected;
|
|
}
|
|
|
|
export function calculateEvalScoreDropPct(
|
|
currentAverage: number | null,
|
|
baselineAverage: number | null,
|
|
): number | null {
|
|
if (
|
|
currentAverage === null ||
|
|
baselineAverage === null ||
|
|
baselineAverage <= 0
|
|
) {
|
|
return null;
|
|
}
|
|
|
|
return ((baselineAverage - currentAverage) / baselineAverage) * 100;
|
|
}
|
|
|
|
export function calculateLlmErrorRate(
|
|
observations: EvalObservation[],
|
|
): number | null {
|
|
if (observations.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
const errorCount = observations.filter(
|
|
(observation) => observation.llmJudgeErrored,
|
|
).length;
|
|
return errorCount / observations.length;
|
|
}
|
|
|
|
function groupObservationsBySkill(
|
|
observations: EvalObservation[],
|
|
): Map<string, EvalObservation[]> {
|
|
const groups = new Map<string, EvalObservation[]>();
|
|
for (const observation of observations) {
|
|
const values = groups.get(observation.skill) ?? [];
|
|
values.push(observation);
|
|
groups.set(observation.skill, values);
|
|
}
|
|
return groups;
|
|
}
|
|
|
|
export class ShadowWorker {
|
|
private archiveFailureCount = 0;
|
|
private lastAlertCheckAt = 0;
|
|
private lastArchiveAt = 0;
|
|
private lastAttributionAt = 0;
|
|
private lastDriftAt = 0;
|
|
private lastEvalAggregateAt = 0;
|
|
private lastPromoteAt = 0;
|
|
private lastReviewAt = 0;
|
|
private loopPromise: Promise<void> | null = null;
|
|
private readonly logger = getLogger();
|
|
private readonly recentEvalObservations: EvalObservation[] = [];
|
|
private stopRequested = false;
|
|
private warnedMissingMongo = false;
|
|
|
|
constructor(
|
|
private readonly garnetClient: GarnetClient,
|
|
private readonly llmClient: LlmClient,
|
|
private readonly conversationStore: ConversationStore,
|
|
private readonly alertSender?: AlertSender,
|
|
private readonly consumerName = `bg-worker-${process.pid}`,
|
|
private readonly mongoConnection?: MongoConnectionLike | undefined,
|
|
) {}
|
|
|
|
async start(): Promise<void> {
|
|
await this.ensureConsumerGroup();
|
|
|
|
if (this.loopPromise) {
|
|
return;
|
|
}
|
|
|
|
this.stopRequested = false;
|
|
this.loopPromise = this.runLoop();
|
|
}
|
|
|
|
async stop(): Promise<void> {
|
|
this.stopRequested = true;
|
|
|
|
if (!this.loopPromise) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS);
|
|
} catch (error) {
|
|
this.logger.warn(
|
|
{ err: error },
|
|
"[ShadowWorker] Stop timed out while waiting for the loop to finish",
|
|
);
|
|
} finally {
|
|
this.loopPromise = null;
|
|
}
|
|
}
|
|
|
|
private async ensureConsumerGroup(): Promise<void> {
|
|
try {
|
|
await this.garnetClient.createConsumerGroup(
|
|
BACKGROUND_STREAM_KEY,
|
|
BACKGROUND_GROUP,
|
|
);
|
|
} catch (error) {
|
|
if (!String(error).includes("BUSYGROUP")) {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
private async runLoop(): Promise<void> {
|
|
while (!this.stopRequested) {
|
|
try {
|
|
await this.runPeriodicChecksIfNeeded();
|
|
|
|
const recoveredPending = await this.recoverPendingOnce();
|
|
if (recoveredPending) {
|
|
continue;
|
|
}
|
|
|
|
const streams = await this.garnetClient.readGroup(
|
|
BACKGROUND_STREAM_KEY,
|
|
BACKGROUND_GROUP,
|
|
this.consumerName,
|
|
">",
|
|
{
|
|
blockMs: READ_BLOCK_MS,
|
|
count: READ_COUNT,
|
|
},
|
|
);
|
|
|
|
if (!streams) {
|
|
continue;
|
|
}
|
|
|
|
for (const stream of streams) {
|
|
for (const message of stream.messages) {
|
|
if (this.stopRequested) {
|
|
return;
|
|
}
|
|
|
|
await this.processMessage(
|
|
message,
|
|
message.deliveriesCounter ?? 1,
|
|
);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
this.logger.error(
|
|
{ err: error },
|
|
"[ShadowWorker] Loop iteration failed",
|
|
);
|
|
await delay(1_000);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async recoverPendingOnce(): Promise<boolean> {
|
|
const pending = await this.garnetClient.getPendingStreamRange(
|
|
BACKGROUND_STREAM_KEY,
|
|
BACKGROUND_GROUP,
|
|
PENDING_BATCH_SIZE,
|
|
);
|
|
const recoverable = pending.filter(
|
|
(entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS,
|
|
);
|
|
|
|
if (recoverable.length === 0) {
|
|
return false;
|
|
}
|
|
|
|
for (const entry of recoverable) {
|
|
if (this.stopRequested) {
|
|
break;
|
|
}
|
|
|
|
await this.recoverPendingEntry(entry);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private async recoverPendingEntry(
|
|
entry: StreamPendingEntry,
|
|
): Promise<void> {
|
|
const nextDeliveryCount = entry.deliveriesCounter + 1;
|
|
const claimed = await this.garnetClient.claimPendingStream(
|
|
BACKGROUND_STREAM_KEY,
|
|
BACKGROUND_GROUP,
|
|
this.consumerName,
|
|
PENDING_IDLE_MS,
|
|
entry.id,
|
|
{
|
|
retryCount: nextDeliveryCount,
|
|
},
|
|
);
|
|
const message = claimed[0];
|
|
|
|
if (!message) {
|
|
return;
|
|
}
|
|
|
|
if (nextDeliveryCount > MAX_DELIVERIES) {
|
|
await this.handleMaxDeliveries(
|
|
message,
|
|
nextDeliveryCount,
|
|
"recovered entry exceeded max deliveries",
|
|
);
|
|
return;
|
|
}
|
|
|
|
await this.processMessage(message, nextDeliveryCount);
|
|
}
|
|
|
|
private async processMessage(
|
|
message: StreamEntry,
|
|
deliveryCount: number,
|
|
): Promise<void> {
|
|
try {
|
|
switch (getJobType(message) as BackgroundJobType) {
|
|
case "shadow-eval":
|
|
await this.processShadowEval(message);
|
|
await this.acknowledge(message.id);
|
|
return;
|
|
case "summarize":
|
|
await this.processSummarize(message);
|
|
await this.acknowledge(message.id);
|
|
return;
|
|
default:
|
|
this.logger.warn(
|
|
{
|
|
messageId: message.id,
|
|
type: getJobType(message),
|
|
},
|
|
"[ShadowWorker] Unknown background job type; acknowledging",
|
|
);
|
|
await this.acknowledge(message.id);
|
|
}
|
|
} catch (error) {
|
|
this.logger.warn(
|
|
{
|
|
err: error,
|
|
deliveryCount,
|
|
messageId: message.id,
|
|
type: getJobType(message),
|
|
},
|
|
"[ShadowWorker] Job processing failed",
|
|
);
|
|
|
|
if (deliveryCount >= MAX_DELIVERIES) {
|
|
await this.handleMaxDeliveries(
|
|
message,
|
|
deliveryCount,
|
|
error instanceof Error ? error.message : String(error),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async processShadowEval(message: StreamEntry): Promise<void> {
|
|
const job = parseShadowEvalJob(message);
|
|
const judgeConfig = getSkillJudgeConfig(job.skill);
|
|
const derivedSnapshot = toDerivedSnapshot(job.derived);
|
|
const abstention = getTraceAbstention(job.process);
|
|
const responseTextForValidation =
|
|
job.response.text.trim()
|
|
? job.response.text
|
|
: abstention?.action.kind === "no-response"
|
|
? "[NO_RESPONSE]"
|
|
: job.response.text;
|
|
const validation = judgeConfig.validator
|
|
? judgeConfig.validator(responseTextForValidation, job.derived)
|
|
: {
|
|
reasons: [],
|
|
status: "pass" as const,
|
|
text: responseTextForValidation,
|
|
};
|
|
const llmJudge = await judgeSkillWithLlm(
|
|
this.llmClient,
|
|
job.skill,
|
|
toEvalCase(job, message.id),
|
|
job.response.text,
|
|
derivedSnapshot,
|
|
);
|
|
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
|
const llmJudgeErrored =
|
|
llmJudge.skipped &&
|
|
llmJudge.reason !== "empty-response" &&
|
|
llmJudge.reason !== "disabled";
|
|
const processJudge = job.process
|
|
? await this.judgeProcess(job, message.id)
|
|
: undefined;
|
|
|
|
this.recordEvalObservation({
|
|
llmJudgeErrored,
|
|
llmOverall,
|
|
recordedAt: Date.now(),
|
|
responseLatencyMs: job.latencyMs ?? null,
|
|
skill: job.skill,
|
|
});
|
|
|
|
const resultFields: Record<string, string> = {
|
|
type: "shadow-eval-result",
|
|
requestId: job.request.requestId,
|
|
chatroomId: job.request.chatroomId,
|
|
userId: job.request.userId,
|
|
skill: job.skill,
|
|
version: String(Math.max(2, job.version)),
|
|
tier: String(derivedSnapshot.tier),
|
|
intent: derivedSnapshot.intent,
|
|
phase: derivedSnapshot.phase,
|
|
responseMode: derivedSnapshot.responseMode,
|
|
validatorStatus: validation.status,
|
|
latencyMs: job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
|
llmOverall: llmOverall === null ? "" : String(llmOverall),
|
|
input: JSON.stringify({
|
|
request: job.request,
|
|
}),
|
|
output: JSON.stringify(job.response),
|
|
derived: JSON.stringify(job.derived),
|
|
classified: JSON.stringify(job.classified),
|
|
validation: JSON.stringify(validation),
|
|
llmJudge: JSON.stringify(llmJudge),
|
|
evaluatedAt: new Date().toISOString(),
|
|
};
|
|
|
|
if (abstention) {
|
|
resultFields.abstentionAction = abstention.action.kind;
|
|
resultFields.abstentionReason = abstention.reason ?? "";
|
|
resultFields.abstentionScore = String(abstention.score);
|
|
}
|
|
|
|
if (job.process) {
|
|
resultFields.process = JSON.stringify(job.process);
|
|
}
|
|
if (processJudge) {
|
|
resultFields.processJudge = JSON.stringify(processJudge);
|
|
}
|
|
|
|
await this.garnetClient.appendStream(
|
|
EVAL_RESULTS_STREAM_KEY,
|
|
resultFields,
|
|
{
|
|
maxLen: getEvalStreamMaxLen(),
|
|
},
|
|
);
|
|
}
|
|
|
|
private async judgeProcess(
|
|
job: ShadowEvalJob,
|
|
messageId: string,
|
|
): Promise<ProcessJudgeResult> {
|
|
if (!job.process) {
|
|
throw new Error("Missing process trace");
|
|
}
|
|
|
|
const judgeConfig = getSkillJudgeConfig(job.skill);
|
|
const ruleResult = judgeProcessRule(
|
|
toEvalCase(job, messageId).expected,
|
|
job.process,
|
|
judgeConfig.toolRegistryFactory(),
|
|
);
|
|
if (
|
|
!getEvalProcessLlmJudgeEnabled() ||
|
|
Math.random() >= getEvalProcessLlmJudgeSamplingRate()
|
|
) {
|
|
return ruleResult;
|
|
}
|
|
|
|
const llmResult = await judgeProcessWithLlm(this.llmClient, job.process, {
|
|
request: job.request,
|
|
finalText: job.response.text,
|
|
});
|
|
|
|
const merged: ProcessJudgeResult = {
|
|
checks: [...ruleResult.checks, ...llmResult.checks],
|
|
passedCount: ruleResult.passedCount + llmResult.passedCount,
|
|
totalCount: ruleResult.totalCount + llmResult.totalCount,
|
|
score: Number(
|
|
(
|
|
(ruleResult.score * ruleResult.totalCount +
|
|
llmResult.score * llmResult.totalCount) /
|
|
Math.max(1, ruleResult.totalCount + llmResult.totalCount)
|
|
).toFixed(3),
|
|
),
|
|
rationale: llmResult.rationale,
|
|
};
|
|
const prmSource = llmResult.prmFeatures ? llmResult : ruleResult;
|
|
if (prmSource.prmFeatures) {
|
|
merged.prmFeatures = prmSource.prmFeatures;
|
|
}
|
|
if (prmSource.prmScore !== undefined) {
|
|
merged.prmScore = prmSource.prmScore;
|
|
}
|
|
if (prmSource.rootCauseStep) {
|
|
merged.rootCauseStep = prmSource.rootCauseStep;
|
|
}
|
|
if (prmSource.rootCauseScoreLoss !== undefined) {
|
|
merged.rootCauseScoreLoss = prmSource.rootCauseScoreLoss;
|
|
}
|
|
|
|
return merged;
|
|
}
|
|
|
|
private async processSummarize(message: StreamEntry): Promise<void> {
|
|
const chatroomId = message.message["chatroomId"];
|
|
const userId = message.message["userId"];
|
|
|
|
if (!chatroomId?.trim() || !userId?.trim()) {
|
|
throw new Error("Missing summarize job identifiers");
|
|
}
|
|
|
|
const [history, existingSummary] = await Promise.all([
|
|
this.conversationStore.getHistory(chatroomId, userId),
|
|
this.conversationStore.getSummary(chatroomId, userId),
|
|
]);
|
|
|
|
if (history.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const summary = await summarizeConversation(
|
|
this.llmClient,
|
|
history,
|
|
existingSummary ?? undefined,
|
|
);
|
|
|
|
await this.conversationStore.setSummary(chatroomId, userId, summary);
|
|
}
|
|
|
|
private async handleMaxDeliveries(
|
|
message: StreamEntry,
|
|
deliveryCount: number,
|
|
lastError: string,
|
|
): Promise<void> {
|
|
const type = getJobType(message);
|
|
|
|
if (type === "summarize") {
|
|
this.logger.warn(
|
|
{
|
|
deliveryCount,
|
|
lastError,
|
|
messageId: message.id,
|
|
},
|
|
"[ShadowWorker] Dropping summarize job after max retries",
|
|
);
|
|
await this.acknowledge(message.id);
|
|
return;
|
|
}
|
|
|
|
await this.garnetClient.appendStream(
|
|
BACKGROUND_DLQ_STREAM_KEY,
|
|
{
|
|
type,
|
|
payload: JSON.stringify(message.message),
|
|
retryCount: String(deliveryCount),
|
|
lastError: lastError.slice(0, 500),
|
|
originalId: message.id,
|
|
},
|
|
{
|
|
maxLen: 1_000,
|
|
},
|
|
);
|
|
await this.acknowledge(message.id);
|
|
}
|
|
|
|
private acknowledge(id: string): Promise<number> {
|
|
return this.garnetClient.acknowledgeStream(
|
|
BACKGROUND_STREAM_KEY,
|
|
BACKGROUND_GROUP,
|
|
id,
|
|
);
|
|
}
|
|
|
|
private recordEvalObservation(observation: EvalObservation): void {
|
|
this.recentEvalObservations.push(observation);
|
|
this.pruneOldObservations(Date.now());
|
|
}
|
|
|
|
private pruneOldObservations(now: number): void {
|
|
const cutoff = now - ALERT_WINDOW_MS;
|
|
let removeCount = 0;
|
|
|
|
for (const observation of this.recentEvalObservations) {
|
|
if (observation.recordedAt >= cutoff) {
|
|
break;
|
|
}
|
|
removeCount += 1;
|
|
}
|
|
|
|
if (removeCount > 0) {
|
|
this.recentEvalObservations.splice(0, removeCount);
|
|
}
|
|
}
|
|
|
|
private async runPeriodicChecksIfNeeded(): Promise<void> {
|
|
const now = Date.now();
|
|
|
|
await this.runAggregationTickIfNeeded(now);
|
|
await this.runArchiveTickIfNeeded(now);
|
|
await this.runAttributionTickIfNeeded(now);
|
|
await this.runDriftTickIfNeeded(now);
|
|
await this.runPromoteTickIfNeeded(now);
|
|
await this.runReviewTickIfNeeded(now);
|
|
|
|
if (!this.alertSender) {
|
|
return;
|
|
}
|
|
|
|
if (now - this.lastAlertCheckAt < ALERT_TICK_MS) {
|
|
return;
|
|
}
|
|
|
|
this.lastAlertCheckAt = now;
|
|
this.pruneOldObservations(now);
|
|
|
|
await this.checkLatencyAlert(now);
|
|
await this.checkEvalScoreDropAlert(now);
|
|
await this.checkLlmErrorRateAlert(now);
|
|
await this.checkOpenCircuitAlerts(now);
|
|
}
|
|
|
|
private async runAggregationTickIfNeeded(now: number): Promise<void> {
|
|
if (now - this.lastEvalAggregateAt < getEvalAggTickMs()) {
|
|
return;
|
|
}
|
|
|
|
this.lastEvalAggregateAt = now;
|
|
const sinceId = await this.garnetClient.getString(EVAL_AGG_CURSOR_KEY);
|
|
const result = await aggregateRecentEvalResults(this.garnetClient, {
|
|
sinceId: sinceId ?? "0-0",
|
|
nowMs: now,
|
|
});
|
|
|
|
if (result.updatedSlices > 0) {
|
|
this.logger.debug(
|
|
{
|
|
lastSeenId: result.lastSeenId,
|
|
updatedSlices: result.updatedSlices,
|
|
},
|
|
"[ShadowWorker] Aggregated shadow eval results",
|
|
);
|
|
}
|
|
}
|
|
|
|
private warnMissingMongoOnce(): void {
|
|
if (this.warnedMissingMongo) {
|
|
return;
|
|
}
|
|
|
|
this.warnedMissingMongo = true;
|
|
this.logger.warn(
|
|
"[ShadowWorker] MongoDB is not configured; archive, drift, and promotion ticks are disabled",
|
|
);
|
|
}
|
|
|
|
private async runArchiveTickIfNeeded(now: number): Promise<void> {
|
|
if (now - this.lastArchiveAt < getEvalArchiveTickMs()) {
|
|
return;
|
|
}
|
|
|
|
this.lastArchiveAt = now;
|
|
if (!this.mongoConnection) {
|
|
this.warnMissingMongoOnce();
|
|
return;
|
|
}
|
|
|
|
const sinceId = await this.garnetClient.getString(EVAL_ARCHIVE_CURSOR_KEY);
|
|
try {
|
|
const result = await archiveEvalResults(
|
|
this.garnetClient,
|
|
this.mongoConnection,
|
|
{
|
|
sinceId: sinceId ?? "0-0",
|
|
nowMs: now,
|
|
},
|
|
);
|
|
const labelSinceId = await this.garnetClient.getString(
|
|
EVAL_REVIEW_LABEL_ARCHIVE_CURSOR_KEY,
|
|
);
|
|
const labelResult = await archiveReviewLabels(
|
|
this.garnetClient,
|
|
this.mongoConnection,
|
|
{
|
|
sinceId: labelSinceId ?? "0-0",
|
|
nowMs: now,
|
|
},
|
|
);
|
|
this.archiveFailureCount = 0;
|
|
if (result.archivedCount > 0) {
|
|
this.logger.info(
|
|
{
|
|
archivedCount: result.archivedCount,
|
|
lastArchivedId: result.lastArchivedId,
|
|
},
|
|
"[ShadowWorker] Archived shadow eval results",
|
|
);
|
|
}
|
|
if (labelResult.archivedCount > 0) {
|
|
this.logger.info(
|
|
{
|
|
archivedCount: labelResult.archivedCount,
|
|
lastArchivedId: labelResult.lastArchivedId,
|
|
},
|
|
"[ShadowWorker] Archived review labels",
|
|
);
|
|
}
|
|
} catch (error) {
|
|
this.archiveFailureCount += 1;
|
|
this.logger.warn(
|
|
{
|
|
err: error,
|
|
failureCount: this.archiveFailureCount,
|
|
},
|
|
"[ShadowWorker] Failed to archive shadow eval results",
|
|
);
|
|
|
|
if (this.archiveFailureCount >= 3 && this.alertSender) {
|
|
await this.alertSender.sendAlert(
|
|
{
|
|
kind: "archive-stalled",
|
|
metric: "shadow_eval.archive_failures",
|
|
severity: "warn",
|
|
suggestedAction: "MongoDB 연결과 eval_archive 인덱스 생성 권한을 확인해 주세요.",
|
|
threshold: 3,
|
|
timestamp: new Date(now).toISOString(),
|
|
value: this.archiveFailureCount,
|
|
window: "consecutive",
|
|
},
|
|
{
|
|
dedupeKey: "lolympus-agent:alert:dedupe:archive-stalled",
|
|
},
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async runDriftTickIfNeeded(now: number): Promise<void> {
|
|
if (now - this.lastDriftAt < getEvalDriftTickMs()) {
|
|
return;
|
|
}
|
|
|
|
this.lastDriftAt = now;
|
|
if (!this.mongoConnection) {
|
|
this.warnMissingMongoOnce();
|
|
return;
|
|
}
|
|
if (!this.alertSender) {
|
|
return;
|
|
}
|
|
|
|
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
|
|
const significantReports = reports.filter((report) => {
|
|
const threshold = getAlertThresholdDriftPct(report.slice.skill);
|
|
return report.recent24hCount >= 30 && report.dropPct >= threshold;
|
|
});
|
|
|
|
for (const report of significantReports) {
|
|
const slice = sliceKeyToString(report.slice);
|
|
const skill = report.slice.skill ?? "onboarding";
|
|
const threshold = getAlertThresholdDriftPct(skill);
|
|
await this.alertSender.sendAlert(
|
|
{
|
|
kind: "eval-drift",
|
|
metric: "shadow_eval.drift.llm_overall_drop_pct",
|
|
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
|
|
skill,
|
|
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
|
|
threshold,
|
|
timestamp: new Date(now).toISOString(),
|
|
value: report.dropPct,
|
|
window: "24h_vs_7d",
|
|
},
|
|
{
|
|
dedupeKey: `lolympus-agent:alert:dedupe:eval-drift:${slice}`,
|
|
},
|
|
);
|
|
}
|
|
}
|
|
|
|
private async runAttributionTickIfNeeded(now: number): Promise<void> {
|
|
if (now - this.lastAttributionAt < getEvalAttributionTickMs()) {
|
|
return;
|
|
}
|
|
|
|
this.lastAttributionAt = now;
|
|
if (!getEvalPrmEnabled()) {
|
|
return;
|
|
}
|
|
if (!this.mongoConnection) {
|
|
this.warnMissingMongoOnce();
|
|
return;
|
|
}
|
|
|
|
const model = loadPrmModel(getEvalPrmWeightsPath());
|
|
if (!model) {
|
|
return;
|
|
}
|
|
|
|
const entries = await getEvalArchiveCollection(this.mongoConnection)
|
|
.find({
|
|
process: { $exists: true },
|
|
})
|
|
.sort({ evaluatedAt: -1 })
|
|
.limit(getEvalArchiveBatchSize())
|
|
.toArray();
|
|
const result = await writeProcessAttributions(
|
|
this.mongoConnection,
|
|
entries,
|
|
model,
|
|
{ nowMs: now },
|
|
);
|
|
|
|
if (result.attributedCount > 0) {
|
|
this.logger.info(
|
|
{
|
|
attributedCount: result.attributedCount,
|
|
modifiedCount: result.modifiedCount,
|
|
upsertedCount: result.upsertedCount,
|
|
},
|
|
"[ShadowWorker] Wrote process attribution entries",
|
|
);
|
|
}
|
|
}
|
|
|
|
private async runPromoteTickIfNeeded(now: number): Promise<void> {
|
|
if (now - this.lastPromoteAt < getEvalPromoteTickMs()) {
|
|
return;
|
|
}
|
|
|
|
this.lastPromoteAt = now;
|
|
if (!this.mongoConnection) {
|
|
this.warnMissingMongoOnce();
|
|
return;
|
|
}
|
|
|
|
const candidates = await selectPromotionCandidates(this.mongoConnection, {
|
|
nowMs: now,
|
|
});
|
|
if (candidates.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const result = await appendCandidatesToJsonl(candidates);
|
|
this.logger.info(
|
|
{
|
|
appendedCount: result.appendedCount,
|
|
outputPath: result.outputPath,
|
|
},
|
|
"[ShadowWorker] Promoted shadow eval candidates",
|
|
);
|
|
}
|
|
|
|
private async runReviewTickIfNeeded(now: number): Promise<void> {
|
|
if (now - this.lastReviewAt < getEvalReviewTickMs()) {
|
|
return;
|
|
}
|
|
|
|
this.lastReviewAt = now;
|
|
const sinceId = await this.garnetClient.getString(EVAL_REVIEW_CURSOR_KEY);
|
|
const result = await routeReviewCandidates(this.garnetClient, {
|
|
sinceId: sinceId ?? "0-0",
|
|
nowMs: now,
|
|
});
|
|
|
|
if (result.enqueuedCount > 0) {
|
|
this.logger.info(
|
|
{
|
|
enqueuedCount: result.enqueuedCount,
|
|
lastSeenId: result.lastSeenId,
|
|
},
|
|
"[ShadowWorker] Routed eval results to review queue",
|
|
);
|
|
}
|
|
}
|
|
|
|
private async checkLatencyAlert(now: number): Promise<void> {
|
|
if (!this.alertSender) {
|
|
return;
|
|
}
|
|
|
|
for (const [skill, observations] of groupObservationsBySkill(
|
|
this.recentEvalObservations,
|
|
)) {
|
|
const latencySamples = observations.flatMap((observation) =>
|
|
observation.responseLatencyMs === null
|
|
? []
|
|
: [observation.responseLatencyMs],
|
|
);
|
|
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS) {
|
|
continue;
|
|
}
|
|
|
|
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
|
const threshold = getAlertThresholdP95LatencyMs(skill);
|
|
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
|
continue;
|
|
}
|
|
|
|
await this.alertSender.sendAlert({
|
|
kind: "latency",
|
|
metric: "shadow_eval.p95_latency_ms",
|
|
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
|
skill,
|
|
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
|
threshold,
|
|
timestamp: new Date(now).toISOString(),
|
|
value: p95LatencyMs,
|
|
window: ALERT_WINDOW_LABEL,
|
|
});
|
|
}
|
|
}
|
|
|
|
private async checkEvalScoreDropAlert(now: number): Promise<void> {
|
|
if (!this.alertSender) {
|
|
return;
|
|
}
|
|
|
|
const baselineAverage = await this.loadBaselineOverallScore();
|
|
for (const [skill, observations] of groupObservationsBySkill(
|
|
this.recentEvalObservations,
|
|
)) {
|
|
const scoreSamples = observations.flatMap((observation) =>
|
|
observation.llmOverall === null ? [] : [observation.llmOverall],
|
|
);
|
|
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
|
continue;
|
|
}
|
|
|
|
const currentAverage =
|
|
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
|
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
|
const threshold = getAlertThresholdEvalScoreDropPct(skill);
|
|
|
|
if (dropPct === null || dropPct <= threshold) {
|
|
continue;
|
|
}
|
|
|
|
await this.alertSender.sendAlert({
|
|
kind: "eval-score-drop",
|
|
metric: "shadow_eval.llm_overall_drop_pct",
|
|
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
|
skill,
|
|
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
|
threshold,
|
|
timestamp: new Date(now).toISOString(),
|
|
value: Number(dropPct.toFixed(3)),
|
|
window: ALERT_WINDOW_LABEL,
|
|
});
|
|
}
|
|
}
|
|
|
|
private async checkLlmErrorRateAlert(now: number): Promise<void> {
|
|
if (!this.alertSender) {
|
|
return;
|
|
}
|
|
|
|
for (const [skill, observations] of groupObservationsBySkill(
|
|
this.recentEvalObservations,
|
|
)) {
|
|
if (observations.length < MIN_ERROR_RATE_OBSERVATIONS) {
|
|
continue;
|
|
}
|
|
|
|
const errorRate = calculateLlmErrorRate(observations);
|
|
const threshold = getAlertThresholdLlmErrorRate(skill);
|
|
if (errorRate === null || errorRate <= threshold) {
|
|
continue;
|
|
}
|
|
|
|
await this.alertSender.sendAlert({
|
|
kind: "llm-error-rate",
|
|
metric: "shadow_eval.llm_error_rate",
|
|
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
|
skill,
|
|
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
|
threshold,
|
|
timestamp: new Date(now).toISOString(),
|
|
value: Number(errorRate.toFixed(4)),
|
|
window: ALERT_WINDOW_LABEL,
|
|
});
|
|
}
|
|
}
|
|
|
|
private async checkOpenCircuitAlerts(now: number): Promise<void> {
|
|
if (!this.alertSender) {
|
|
return;
|
|
}
|
|
|
|
const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*");
|
|
|
|
for (const key of circuitKeys) {
|
|
const record = await this.garnetClient.getHash(key);
|
|
if (record["state"] !== "open") {
|
|
continue;
|
|
}
|
|
|
|
const endpointHash = key.split(":").at(-1) ?? "unknown";
|
|
const failureCount = Number(record["failureCount"]);
|
|
|
|
await this.alertSender.sendAlert(
|
|
{
|
|
kind: "endpoint-down",
|
|
metric: "llm.circuit.open",
|
|
severity: "critical",
|
|
suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.",
|
|
threshold: 0,
|
|
timestamp: new Date(now).toISOString(),
|
|
value: Number.isFinite(failureCount) ? failureCount : 1,
|
|
window: ALERT_WINDOW_LABEL,
|
|
},
|
|
{
|
|
dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`,
|
|
},
|
|
);
|
|
}
|
|
}
|
|
|
|
private async loadBaselineOverallScore(): Promise<number | null> {
|
|
const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY);
|
|
const redisParsed = parseBaselineOverallFromRaw(redisBaseline);
|
|
if (redisParsed !== null) {
|
|
return redisParsed;
|
|
}
|
|
|
|
const filePath = EVAL_BASELINE_FILES
|
|
.map((candidate) => resolve(process.cwd(), candidate))
|
|
.find((candidate) => existsSync(candidate));
|
|
if (!filePath) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const fileRaw = await readFile(filePath, "utf8");
|
|
return parseBaselineOverallFromRaw(fileRaw);
|
|
} catch (error) {
|
|
this.logger.warn(
|
|
{ err: error, filePath },
|
|
"[ShadowWorker] Failed to read baseline file",
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
|
|
function parseBaselineOverallFromRaw(raw: string | null): number | null {
|
|
if (!raw?.trim()) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
const parsed = JSON.parse(raw) as Partial<EvalBaselineFile>;
|
|
const value = parsed.summary?.llmAverageOverall;
|
|
return typeof value === "number" ? value : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|