Add Slack alerts and skill-aware monitoring
This commit is contained in:
1 parent
b63985694a
commit
f1b30d716d
19 files changed
+1288
-112
No files matched your search
+105
-64
@@ -112,6 +112,7 @@ interface EvalObservation {
|
||||
llmOverall: number | null;
|
||||
recordedAt: number;
|
||||
responseLatencyMs: number | null;
|
||||
skill: string;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
@@ -331,6 +332,18 @@ export function calculateLlmErrorRate(
|
||||
return errorCount / observations.length;
|
||||
}
|
||||
|
||||
function groupObservationsBySkill(
|
||||
observations: EvalObservation[],
|
||||
): Map<string, EvalObservation[]> {
|
||||
const groups = new Map<string, EvalObservation[]>();
|
||||
for (const observation of observations) {
|
||||
const values = groups.get(observation.skill) ?? [];
|
||||
values.push(observation);
|
||||
groups.set(observation.skill, values);
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
export class ShadowWorker {
|
||||
private archiveFailureCount = 0;
|
||||
private lastAlertCheckAt = 0;
|
||||
@@ -586,6 +599,7 @@ export class ShadowWorker {
|
||||
llmOverall,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: job.latencyMs ?? null,
|
||||
skill: job.skill,
|
||||
});
|
||||
|
||||
const resultFields: Record<string, string> = {
|
||||
@@ -938,17 +952,22 @@ export class ShadowWorker {
|
||||
return;
|
||||
}
|
||||
|
||||
const threshold = getAlertThresholdDriftPct();
|
||||
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
|
||||
const significantReports = reports.filter((report) => report.significant);
|
||||
const significantReports = reports.filter((report) => {
|
||||
const threshold = getAlertThresholdDriftPct(report.slice.skill);
|
||||
return report.recent24hCount >= 30 && report.dropPct >= threshold;
|
||||
});
|
||||
|
||||
for (const report of significantReports) {
|
||||
const slice = sliceKeyToString(report.slice);
|
||||
const skill = report.slice.skill ?? "onboarding";
|
||||
const threshold = getAlertThresholdDriftPct(skill);
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "eval-drift",
|
||||
metric: "shadow_eval.drift.llm_overall_drop_pct",
|
||||
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
@@ -1059,29 +1078,40 @@ export class ShadowWorker {
|
||||
}
|
||||
|
||||
private async checkLatencyAlert(now: number): Promise<void> {
|
||||
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
|
||||
);
|
||||
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
||||
const threshold = getAlertThresholdP95LatencyMs();
|
||||
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
||||
return;
|
||||
}
|
||||
for (const [skill, observations] of groupObservationsBySkill(
|
||||
this.recentEvalObservations,
|
||||
)) {
|
||||
const latencySamples = observations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null
|
||||
? []
|
||||
: [observation.responseLatencyMs],
|
||||
);
|
||||
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: p95LatencyMs,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
||||
const threshold = getAlertThresholdP95LatencyMs(skill);
|
||||
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: p95LatencyMs,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async checkEvalScoreDropAlert(now: number): Promise<void> {
|
||||
@@ -1089,59 +1119,70 @@ export class ShadowWorker {
|
||||
return;
|
||||
}
|
||||
|
||||
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.llmOverall === null ? [] : [observation.llmOverall],
|
||||
);
|
||||
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentAverage =
|
||||
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
||||
const baselineAverage = await this.loadBaselineOverallScore();
|
||||
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
||||
const threshold = getAlertThresholdEvalScoreDropPct();
|
||||
for (const [skill, observations] of groupObservationsBySkill(
|
||||
this.recentEvalObservations,
|
||||
)) {
|
||||
const scoreSamples = observations.flatMap((observation) =>
|
||||
observation.llmOverall === null ? [] : [observation.llmOverall],
|
||||
);
|
||||
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dropPct === null || dropPct <= threshold) {
|
||||
return;
|
||||
const currentAverage =
|
||||
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
||||
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
||||
const threshold = getAlertThresholdEvalScoreDropPct(skill);
|
||||
|
||||
if (dropPct === null || dropPct <= threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "eval-score-drop",
|
||||
metric: "shadow_eval.llm_overall_drop_pct",
|
||||
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(dropPct.toFixed(3)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "eval-score-drop",
|
||||
metric: "shadow_eval.llm_overall_drop_pct",
|
||||
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(dropPct.toFixed(3)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkLlmErrorRateAlert(now: number): Promise<void> {
|
||||
if (
|
||||
!this.alertSender ||
|
||||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
|
||||
) {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
|
||||
const threshold = getAlertThresholdLlmErrorRate();
|
||||
if (errorRate === null || errorRate <= threshold) {
|
||||
return;
|
||||
}
|
||||
for (const [skill, observations] of groupObservationsBySkill(
|
||||
this.recentEvalObservations,
|
||||
)) {
|
||||
if (observations.length < MIN_ERROR_RATE_OBSERVATIONS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "llm-error-rate",
|
||||
metric: "shadow_eval.llm_error_rate",
|
||||
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(errorRate.toFixed(4)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
const errorRate = calculateLlmErrorRate(observations);
|
||||
const threshold = getAlertThresholdLlmErrorRate(skill);
|
||||
if (errorRate === null || errorRate <= threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "llm-error-rate",
|
||||
metric: "shadow_eval.llm_error_rate",
|
||||
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(errorRate.toFixed(4)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async checkOpenCircuitAlerts(now: number): Promise<void> {
|
||||
|
||||
Reference in new issue
Block a user