Add eval runner and tool orchestration
This commit is contained in:
1 parent
4294f3c92e
commit
5579db54da
66 files changed
+6162
-242
No files matched your search
@@ -0,0 +1,741 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { AgentRequest, AgentResponse } from "@agent/types";
|
||||
import type {
|
||||
GarnetClient,
|
||||
StreamEntry,
|
||||
StreamPendingEntry,
|
||||
} from "@common/garnet";
|
||||
import { summarizeConversation } from "@conv/summarizer";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { getLogger } from "@obs/logger";
|
||||
import type { AlertSender } from "@obs/alert";
|
||||
import { validateOnboardingResponse } from "@skills/onboarding/runtime/validator";
|
||||
import type {
|
||||
ClassifiedIntent,
|
||||
DerivedState,
|
||||
} from "@skills/onboarding/state/onboarding-state";
|
||||
import type { ConversationStore } from "@conv/conversation-store";
|
||||
import {
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getEvalStreamMaxLen,
|
||||
} from "../config.js";
|
||||
import { judgeOnboardingWithLlm } from "./judges/llm-judge.js";
|
||||
import type {
|
||||
EvalBaselineFile,
|
||||
EvalCaseDefinition,
|
||||
EvalDerivedSnapshot,
|
||||
} from "./types.js";
|
||||
|
||||
const BACKGROUND_STREAM_KEY = "lolympus-agent:background";
|
||||
const BACKGROUND_DLQ_STREAM_KEY = "lolympus-agent:background:dlq";
|
||||
const BACKGROUND_GROUP = "bg-worker";
|
||||
const EVAL_BASELINE_KEY = "lolympus-agent:eval:baseline";
|
||||
const EVAL_RESULTS_STREAM_KEY = "lolympus-agent:eval:results";
|
||||
const EVAL_BASELINE_FILE = "eval-reports/baseline.json";
|
||||
const ALERT_WINDOW_MS = 5 * 60_000;
|
||||
const ALERT_WINDOW_LABEL = "5m";
|
||||
const ALERT_TICK_MS = 60_000;
|
||||
const MAX_DELIVERIES = 3;
|
||||
const MIN_ERROR_RATE_OBSERVATIONS = 5;
|
||||
const MIN_LATENCY_OBSERVATIONS = 5;
|
||||
const MIN_SCORE_OBSERVATIONS = 3;
|
||||
const PENDING_IDLE_MS = 1_000;
|
||||
const READ_BLOCK_MS = 5_000;
|
||||
const READ_COUNT = 10;
|
||||
const PENDING_BATCH_SIZE = 20;
|
||||
const STOP_TIMEOUT_MS = 10_000;
|
||||
|
||||
type BackgroundJobType = "shadow-eval" | "summarize";
|
||||
|
||||
interface ShadowEvalJob {
|
||||
classified: ClassifiedIntent;
|
||||
createdAt: string;
|
||||
derived: DerivedState;
|
||||
latencyMs?: number | undefined;
|
||||
request: AgentRequest;
|
||||
response: AgentResponse;
|
||||
}
|
||||
|
||||
interface EvalObservation {
|
||||
llmJudgeErrored: boolean;
|
||||
llmOverall: number | null;
|
||||
recordedAt: number;
|
||||
responseLatencyMs: number | null;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
function promiseWithTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
): Promise<T> {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise<T>((_, reject) => {
|
||||
setTimeout(() => {
|
||||
reject(new Error(`Timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
function getJobType(message: StreamEntry): string {
|
||||
return message.message["type"] ?? "unknown";
|
||||
}
|
||||
|
||||
function parseOptionalNumberField(value: string | undefined): number | undefined {
|
||||
if (!value?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseJsonField<T>(
|
||||
value: string | undefined,
|
||||
fieldName: string,
|
||||
): T {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`Missing ${fieldName} field`);
|
||||
}
|
||||
|
||||
return JSON.parse(value) as T;
|
||||
}
|
||||
|
||||
function toDerivedSnapshot(derived: DerivedState): EvalDerivedSnapshot {
|
||||
return {
|
||||
allowShortResponse: derived.allowShortResponse,
|
||||
intent: derived.intent.kind,
|
||||
phase: derived.phase.kind,
|
||||
responseMode: derived.responseMode,
|
||||
shouldRespond: derived.shouldRespond,
|
||||
suggestedMaxTokens: derived.suggestedMaxTokens,
|
||||
tier: derived.tier,
|
||||
};
|
||||
}
|
||||
|
||||
function toEvalCase(job: ShadowEvalJob, messageId: string): EvalCaseDefinition {
|
||||
return {
|
||||
id: `shadow:${messageId}`,
|
||||
lineNumber: 0,
|
||||
input: {
|
||||
request: job.request,
|
||||
},
|
||||
expected: {},
|
||||
};
|
||||
}
|
||||
|
||||
function parseShadowEvalJob(message: StreamEntry): ShadowEvalJob {
|
||||
const input = parseJsonField<{ request: AgentRequest }>(
|
||||
message.message["input"],
|
||||
"input",
|
||||
);
|
||||
|
||||
return {
|
||||
classified: parseJsonField<ClassifiedIntent>(
|
||||
message.message["classified"],
|
||||
"classified",
|
||||
),
|
||||
createdAt: message.message["createdAt"] ?? new Date().toISOString(),
|
||||
derived: parseJsonField<DerivedState>(
|
||||
message.message["derived"],
|
||||
"derived",
|
||||
),
|
||||
latencyMs: parseOptionalNumberField(message.message["latencyMs"]),
|
||||
request: input.request,
|
||||
response: parseJsonField<AgentResponse>(
|
||||
message.message["output"],
|
||||
"output",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function calculatePercentile(
|
||||
values: number[],
|
||||
percentile: number,
|
||||
): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const index = Math.min(
|
||||
sorted.length - 1,
|
||||
Math.max(0, Math.ceil(sorted.length * percentile) - 1),
|
||||
);
|
||||
const selected = sorted[index];
|
||||
return selected === undefined ? null : selected;
|
||||
}
|
||||
|
||||
export function calculateEvalScoreDropPct(
|
||||
currentAverage: number | null,
|
||||
baselineAverage: number | null,
|
||||
): number | null {
|
||||
if (
|
||||
currentAverage === null ||
|
||||
baselineAverage === null ||
|
||||
baselineAverage <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return ((baselineAverage - currentAverage) / baselineAverage) * 100;
|
||||
}
|
||||
|
||||
export function calculateLlmErrorRate(
|
||||
observations: EvalObservation[],
|
||||
): number | null {
|
||||
if (observations.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const errorCount = observations.filter(
|
||||
(observation) => observation.llmJudgeErrored,
|
||||
).length;
|
||||
return errorCount / observations.length;
|
||||
}
|
||||
|
||||
export class ShadowWorker {
|
||||
private lastAggregateAt = 0;
|
||||
private loopPromise: Promise<void> | null = null;
|
||||
private readonly logger = getLogger();
|
||||
private readonly recentEvalObservations: EvalObservation[] = [];
|
||||
private stopRequested = false;
|
||||
|
||||
constructor(
|
||||
private readonly garnetClient: GarnetClient,
|
||||
private readonly llmClient: LlmClient,
|
||||
private readonly conversationStore: ConversationStore,
|
||||
private readonly alertSender?: AlertSender,
|
||||
private readonly consumerName = `bg-worker-${process.pid}`,
|
||||
) {}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.ensureConsumerGroup();
|
||||
|
||||
if (this.loopPromise) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.stopRequested = false;
|
||||
this.loopPromise = this.runLoop();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopRequested = true;
|
||||
|
||||
if (!this.loopPromise) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await promiseWithTimeout(this.loopPromise, STOP_TIMEOUT_MS);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{ err: error },
|
||||
"[ShadowWorker] Stop timed out while waiting for the loop to finish",
|
||||
);
|
||||
} finally {
|
||||
this.loopPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureConsumerGroup(): Promise<void> {
|
||||
try {
|
||||
await this.garnetClient.createConsumerGroup(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!String(error).includes("BUSYGROUP")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runLoop(): Promise<void> {
|
||||
while (!this.stopRequested) {
|
||||
try {
|
||||
await this.runPeriodicChecksIfNeeded();
|
||||
|
||||
const recoveredPending = await this.recoverPendingOnce();
|
||||
if (recoveredPending) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const streams = await this.garnetClient.readGroup(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
this.consumerName,
|
||||
">",
|
||||
{
|
||||
blockMs: READ_BLOCK_MS,
|
||||
count: READ_COUNT,
|
||||
},
|
||||
);
|
||||
|
||||
if (!streams) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const stream of streams) {
|
||||
for (const message of stream.messages) {
|
||||
if (this.stopRequested) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.processMessage(
|
||||
message,
|
||||
message.deliveriesCounter ?? 1,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ err: error },
|
||||
"[ShadowWorker] Loop iteration failed",
|
||||
);
|
||||
await delay(1_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async recoverPendingOnce(): Promise<boolean> {
|
||||
const pending = await this.garnetClient.getPendingStreamRange(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
PENDING_BATCH_SIZE,
|
||||
);
|
||||
const recoverable = pending.filter(
|
||||
(entry) => entry.millisecondsSinceLastDelivery >= PENDING_IDLE_MS,
|
||||
);
|
||||
|
||||
if (recoverable.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const entry of recoverable) {
|
||||
if (this.stopRequested) {
|
||||
break;
|
||||
}
|
||||
|
||||
await this.recoverPendingEntry(entry);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async recoverPendingEntry(
|
||||
entry: StreamPendingEntry,
|
||||
): Promise<void> {
|
||||
const nextDeliveryCount = entry.deliveriesCounter + 1;
|
||||
const claimed = await this.garnetClient.claimPendingStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
this.consumerName,
|
||||
PENDING_IDLE_MS,
|
||||
entry.id,
|
||||
{
|
||||
retryCount: nextDeliveryCount,
|
||||
},
|
||||
);
|
||||
const message = claimed[0];
|
||||
|
||||
if (!message) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (nextDeliveryCount > MAX_DELIVERIES) {
|
||||
await this.handleMaxDeliveries(
|
||||
message,
|
||||
nextDeliveryCount,
|
||||
"recovered entry exceeded max deliveries",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.processMessage(message, nextDeliveryCount);
|
||||
}
|
||||
|
||||
private async processMessage(
|
||||
message: StreamEntry,
|
||||
deliveryCount: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
switch (getJobType(message) as BackgroundJobType) {
|
||||
case "shadow-eval":
|
||||
await this.processShadowEval(message);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
case "summarize":
|
||||
await this.processSummarize(message);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
default:
|
||||
this.logger.warn(
|
||||
{
|
||||
messageId: message.id,
|
||||
type: getJobType(message),
|
||||
},
|
||||
"[ShadowWorker] Unknown background job type; acknowledging",
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{
|
||||
err: error,
|
||||
deliveryCount,
|
||||
messageId: message.id,
|
||||
type: getJobType(message),
|
||||
},
|
||||
"[ShadowWorker] Job processing failed",
|
||||
);
|
||||
|
||||
if (deliveryCount >= MAX_DELIVERIES) {
|
||||
await this.handleMaxDeliveries(
|
||||
message,
|
||||
deliveryCount,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async processShadowEval(message: StreamEntry): Promise<void> {
|
||||
const job = parseShadowEvalJob(message);
|
||||
const validation = validateOnboardingResponse(
|
||||
job.response.text,
|
||||
job.derived,
|
||||
{
|
||||
allowShortResponse: job.derived.allowShortResponse,
|
||||
},
|
||||
);
|
||||
const llmJudge = await judgeOnboardingWithLlm(
|
||||
this.llmClient,
|
||||
toEvalCase(job, message.id),
|
||||
job.response.text,
|
||||
toDerivedSnapshot(job.derived),
|
||||
);
|
||||
const llmOverall = llmJudge.skipped ? null : llmJudge.scores.overall;
|
||||
const llmJudgeErrored =
|
||||
llmJudge.skipped &&
|
||||
llmJudge.reason !== "empty-response" &&
|
||||
llmJudge.reason !== "disabled";
|
||||
|
||||
this.recordEvalObservation({
|
||||
llmJudgeErrored,
|
||||
llmOverall,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: job.latencyMs ?? null,
|
||||
});
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
EVAL_RESULTS_STREAM_KEY,
|
||||
{
|
||||
type: "shadow-eval-result",
|
||||
requestId: job.request.requestId,
|
||||
chatroomId: job.request.chatroomId,
|
||||
userId: job.request.userId,
|
||||
tier: String(job.derived.tier),
|
||||
intent: job.derived.intent.kind,
|
||||
responseMode: job.derived.responseMode,
|
||||
validatorStatus: validation.status,
|
||||
latencyMs:
|
||||
job.latencyMs !== undefined ? String(job.latencyMs) : "",
|
||||
llmOverall:
|
||||
llmOverall === null ? "" : String(llmOverall),
|
||||
input: JSON.stringify({
|
||||
request: job.request,
|
||||
}),
|
||||
output: JSON.stringify(job.response),
|
||||
derived: JSON.stringify(job.derived),
|
||||
classified: JSON.stringify(job.classified),
|
||||
validation: JSON.stringify(validation),
|
||||
llmJudge: JSON.stringify(llmJudge),
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
maxLen: getEvalStreamMaxLen(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private async processSummarize(message: StreamEntry): Promise<void> {
|
||||
const chatroomId = message.message["chatroomId"];
|
||||
const userId = message.message["userId"];
|
||||
|
||||
if (!chatroomId?.trim() || !userId?.trim()) {
|
||||
throw new Error("Missing summarize job identifiers");
|
||||
}
|
||||
|
||||
const [history, existingSummary] = await Promise.all([
|
||||
this.conversationStore.getHistory(chatroomId, userId),
|
||||
this.conversationStore.getSummary(chatroomId, userId),
|
||||
]);
|
||||
|
||||
if (history.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const summary = await summarizeConversation(
|
||||
this.llmClient,
|
||||
history,
|
||||
existingSummary ?? undefined,
|
||||
);
|
||||
|
||||
await this.conversationStore.setSummary(chatroomId, userId, summary);
|
||||
}
|
||||
|
||||
private async handleMaxDeliveries(
|
||||
message: StreamEntry,
|
||||
deliveryCount: number,
|
||||
lastError: string,
|
||||
): Promise<void> {
|
||||
const type = getJobType(message);
|
||||
|
||||
if (type === "summarize") {
|
||||
this.logger.warn(
|
||||
{
|
||||
deliveryCount,
|
||||
lastError,
|
||||
messageId: message.id,
|
||||
},
|
||||
"[ShadowWorker] Dropping summarize job after max retries",
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.garnetClient.appendStream(
|
||||
BACKGROUND_DLQ_STREAM_KEY,
|
||||
{
|
||||
type,
|
||||
payload: JSON.stringify(message.message),
|
||||
retryCount: String(deliveryCount),
|
||||
lastError: lastError.slice(0, 500),
|
||||
originalId: message.id,
|
||||
},
|
||||
{
|
||||
maxLen: 1_000,
|
||||
},
|
||||
);
|
||||
await this.acknowledge(message.id);
|
||||
}
|
||||
|
||||
private acknowledge(id: string): Promise<number> {
|
||||
return this.garnetClient.acknowledgeStream(
|
||||
BACKGROUND_STREAM_KEY,
|
||||
BACKGROUND_GROUP,
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
private recordEvalObservation(observation: EvalObservation): void {
|
||||
this.recentEvalObservations.push(observation);
|
||||
this.pruneOldObservations(Date.now());
|
||||
}
|
||||
|
||||
private pruneOldObservations(now: number): void {
|
||||
const cutoff = now - ALERT_WINDOW_MS;
|
||||
let removeCount = 0;
|
||||
|
||||
for (const observation of this.recentEvalObservations) {
|
||||
if (observation.recordedAt >= cutoff) {
|
||||
break;
|
||||
}
|
||||
removeCount += 1;
|
||||
}
|
||||
|
||||
if (removeCount > 0) {
|
||||
this.recentEvalObservations.splice(0, removeCount);
|
||||
}
|
||||
}
|
||||
|
||||
private async runPeriodicChecksIfNeeded(): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (now - this.lastAggregateAt < ALERT_TICK_MS) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastAggregateAt = now;
|
||||
this.pruneOldObservations(now);
|
||||
|
||||
await this.checkLatencyAlert(now);
|
||||
await this.checkEvalScoreDropAlert(now);
|
||||
await this.checkLlmErrorRateAlert(now);
|
||||
await this.checkOpenCircuitAlerts(now);
|
||||
}
|
||||
|
||||
private async checkLatencyAlert(now: number): Promise<void> {
|
||||
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
|
||||
);
|
||||
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
||||
const threshold = getAlertThresholdP95LatencyMs();
|
||||
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: p95LatencyMs,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkEvalScoreDropAlert(now: number): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.llmOverall === null ? [] : [observation.llmOverall],
|
||||
);
|
||||
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentAverage =
|
||||
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
||||
const baselineAverage = await this.loadBaselineOverallScore();
|
||||
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
||||
const threshold = getAlertThresholdEvalScoreDropPct();
|
||||
|
||||
if (dropPct === null || dropPct <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "eval-score-drop",
|
||||
metric: "shadow_eval.llm_overall_drop_pct",
|
||||
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(dropPct.toFixed(3)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkLlmErrorRateAlert(now: number): Promise<void> {
|
||||
if (
|
||||
!this.alertSender ||
|
||||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
|
||||
const threshold = getAlertThresholdLlmErrorRate();
|
||||
if (errorRate === null || errorRate <= threshold) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "llm-error-rate",
|
||||
metric: "shadow_eval.llm_error_rate",
|
||||
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(errorRate.toFixed(4)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkOpenCircuitAlerts(now: number): Promise<void> {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const circuitKeys = await this.garnetClient.scanKeys("lolympus-agent:circuit:*");
|
||||
|
||||
for (const key of circuitKeys) {
|
||||
const record = await this.garnetClient.getHash(key);
|
||||
if (record["state"] !== "open") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const endpointHash = key.split(":").at(-1) ?? "unknown";
|
||||
const failureCount = Number(record["failureCount"]);
|
||||
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "endpoint-down",
|
||||
metric: "llm.circuit.open",
|
||||
severity: "critical",
|
||||
suggestedAction: "해당 upstream endpoint 상태와 인증/네트워크 연결을 확인해 주세요.",
|
||||
threshold: 0,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number.isFinite(failureCount) ? failureCount : 1,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
},
|
||||
{
|
||||
dedupeKey: `lolympus-agent:alert:dedupe:endpoint:${endpointHash}`,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadBaselineOverallScore(): Promise<number | null> {
|
||||
const redisBaseline = await this.garnetClient.getString(EVAL_BASELINE_KEY);
|
||||
const redisParsed = parseBaselineOverallFromRaw(redisBaseline);
|
||||
if (redisParsed !== null) {
|
||||
return redisParsed;
|
||||
}
|
||||
|
||||
const filePath = resolve(process.cwd(), EVAL_BASELINE_FILE);
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const fileRaw = await readFile(filePath, "utf8");
|
||||
return parseBaselineOverallFromRaw(fileRaw);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{ err: error, filePath },
|
||||
"[ShadowWorker] Failed to read baseline file",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseBaselineOverallFromRaw(raw: string | null): number | null {
|
||||
if (!raw?.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as Partial<EvalBaselineFile>;
|
||||
const value = parsed.summary?.llmAverageOverall;
|
||||
return typeof value === "number" ? value : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user