Add Slack alerts and skill-aware monitoring

This commit is contained in:
dahyeon committed 2026-05-18 19:11:05 -07:00
1 parent b63985694a
commit f1b30d716d
19 files changed
+1288 -112

No files matched your search

+105 -64
View File
@@ -112,6 +112,7 @@ interface EvalObservation {
llmOverall: number | null;
recordedAt: number;
responseLatencyMs: number | null;
skill: string;
}
function delay(ms: number): Promise<void> {
@@ -331,6 +332,18 @@ export function calculateLlmErrorRate(
return errorCount / observations.length;
}
function groupObservationsBySkill(
observations: EvalObservation[],
): Map<string, EvalObservation[]> {
const groups = new Map<string, EvalObservation[]>();
for (const observation of observations) {
const values = groups.get(observation.skill) ?? [];
values.push(observation);
groups.set(observation.skill, values);
}
return groups;
}
export class ShadowWorker {
private archiveFailureCount = 0;
private lastAlertCheckAt = 0;
@@ -586,6 +599,7 @@ export class ShadowWorker {
llmOverall,
recordedAt: Date.now(),
responseLatencyMs: job.latencyMs ?? null,
skill: job.skill,
});
const resultFields: Record<string, string> = {
@@ -938,17 +952,22 @@ export class ShadowWorker {
return;
}
const threshold = getAlertThresholdDriftPct();
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
const significantReports = reports.filter((report) => report.significant);
const significantReports = reports.filter((report) => {
const threshold = getAlertThresholdDriftPct(report.slice.skill);
return report.recent24hCount >= 30 && report.dropPct >= threshold;
});
for (const report of significantReports) {
const slice = sliceKeyToString(report.slice);
const skill = report.slice.skill ?? "onboarding";
const threshold = getAlertThresholdDriftPct(skill);
await this.alertSender.sendAlert(
{
kind: "eval-drift",
metric: "shadow_eval.drift.llm_overall_drop_pct",
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
threshold,
timestamp: new Date(now).toISOString(),
@@ -1059,29 +1078,40 @@ export class ShadowWorker {
}
private async checkLatencyAlert(now: number): Promise<void> {
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
);
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
if (!this.alertSender) {
return;
}
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
const threshold = getAlertThresholdP95LatencyMs();
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
return;
}
for (const [skill, observations] of groupObservationsBySkill(
this.recentEvalObservations,
)) {
const latencySamples = observations.flatMap((observation) =>
observation.responseLatencyMs === null
? []
: [observation.responseLatencyMs],
);
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS) {
continue;
}
await this.alertSender.sendAlert({
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: p95LatencyMs,
window: ALERT_WINDOW_LABEL,
});
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
const threshold = getAlertThresholdP95LatencyMs(skill);
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
continue;
}
await this.alertSender.sendAlert({
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: p95LatencyMs,
window: ALERT_WINDOW_LABEL,
});
}
}
private async checkEvalScoreDropAlert(now: number): Promise<void> {
@@ -1089,59 +1119,70 @@ export class ShadowWorker {
return;
}
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
observation.llmOverall === null ? [] : [observation.llmOverall],
);
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
return;
}
const currentAverage =
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
const baselineAverage = await this.loadBaselineOverallScore();
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
const threshold = getAlertThresholdEvalScoreDropPct();
for (const [skill, observations] of groupObservationsBySkill(
this.recentEvalObservations,
)) {
const scoreSamples = observations.flatMap((observation) =>
observation.llmOverall === null ? [] : [observation.llmOverall],
);
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
continue;
}
if (dropPct === null || dropPct <= threshold) {
return;
const currentAverage =
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
const threshold = getAlertThresholdEvalScoreDropPct(skill);
if (dropPct === null || dropPct <= threshold) {
continue;
}
await this.alertSender.sendAlert({
kind: "eval-score-drop",
metric: "shadow_eval.llm_overall_drop_pct",
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(dropPct.toFixed(3)),
window: ALERT_WINDOW_LABEL,
});
}
await this.alertSender.sendAlert({
kind: "eval-score-drop",
metric: "shadow_eval.llm_overall_drop_pct",
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(dropPct.toFixed(3)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkLlmErrorRateAlert(now: number): Promise<void> {
if (
!this.alertSender ||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
) {
if (!this.alertSender) {
return;
}
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
const threshold = getAlertThresholdLlmErrorRate();
if (errorRate === null || errorRate <= threshold) {
return;
}
for (const [skill, observations] of groupObservationsBySkill(
this.recentEvalObservations,
)) {
if (observations.length < MIN_ERROR_RATE_OBSERVATIONS) {
continue;
}
await this.alertSender.sendAlert({
kind: "llm-error-rate",
metric: "shadow_eval.llm_error_rate",
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(errorRate.toFixed(4)),
window: ALERT_WINDOW_LABEL,
});
const errorRate = calculateLlmErrorRate(observations);
const threshold = getAlertThresholdLlmErrorRate(skill);
if (errorRate === null || errorRate <= threshold) {
continue;
}
await this.alertSender.sendAlert({
kind: "llm-error-rate",
metric: "shadow_eval.llm_error_rate",
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(errorRate.toFixed(4)),
window: ALERT_WINDOW_LABEL,
});
}
}
private async checkOpenCircuitAlerts(now: number): Promise<void> {