Add Slack alerts and skill-aware monitoring

This commit is contained in:
dahyeon committed 2026-05-18 19:11:05 -07:00
1 parent b63985694a
commit f1b30d716d
19 files changed
+1288 -112

No files matched your search

+69 -8
View File
@@ -29,6 +29,47 @@ function getRatioNumber(
: fallback;
}
function normalizeSkillEnvSuffix(skill: string | undefined): string | undefined {
const suffix = skill?.trim().toUpperCase().replace(/[^A-Z0-9]/gu, "_");
return suffix || undefined;
}
function getPositiveNumberForSkill(
envKey: string,
skill: string | undefined,
fallback: number,
): number {
const skillSuffix = normalizeSkillEnvSuffix(skill);
if (skillSuffix) {
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
if (raw) {
const parsed = Number(raw);
if (Number.isFinite(parsed) && parsed > 0) {
return parsed;
}
}
}
return getPositiveNumber(envKey, fallback);
}
function getRatioNumberForSkill(
envKey: string,
skill: string | undefined,
fallback: number,
): number {
const skillSuffix = normalizeSkillEnvSuffix(skill);
if (skillSuffix) {
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
if (raw) {
const parsed = Number(raw);
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
return parsed;
}
}
}
return getRatioNumber(envKey, fallback);
}
function getBoolean(envKey: string, fallback: boolean): boolean {
const raw = process.env[envKey]?.trim().toLowerCase();
if (!raw) return fallback;
@@ -539,22 +580,42 @@ export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
}
}
export function getAlertThresholdP95LatencyMs(): number {
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
export function getAlertThresholdP95LatencyMs(skill?: string): number {
return getPositiveNumberForSkill(
"ALERT_THRESHOLD_P95_LATENCY_MS",
skill,
3_000,
);
}
export function getAlertThresholdEvalScoreDropPct(): number {
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
export function getAlertThresholdEvalScoreDropPct(skill?: string): number {
return getPositiveNumberForSkill(
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT",
skill,
10,
);
}
export function getAlertThresholdLlmErrorRate(): number {
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
export function getAlertThresholdLlmErrorRate(skill?: string): number {
return getRatioNumberForSkill(
"ALERT_THRESHOLD_LLM_ERROR_RATE",
skill,
0.1,
);
}
export function getAlertThresholdDriftPct(): number {
return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15);
export function getAlertThresholdDriftPct(skill?: string): number {
return getPositiveNumberForSkill("ALERT_THRESHOLD_DRIFT_PCT", skill, 15);
}
export function getAlertRateLimitSec(): number {
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
}
export function getSlackAlertWebhookUrl(): string | undefined {
return process.env["SLACK_ALERT_WEBHOOK_URL"]?.trim() || undefined;
}
export function getSlackWebhookTimeoutMs(): number {
return getPositiveNumber("SLACK_WEBHOOK_TIMEOUT_MS", 3_000);
}