Add Slack alerts and skill-aware monitoring
This commit is contained in:
1 parent
b63985694a
commit
f1b30d716d
19 files changed
+1288
-112
No files matched your search
+69
-8
@@ -29,6 +29,47 @@ function getRatioNumber(
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function normalizeSkillEnvSuffix(skill: string | undefined): string | undefined {
|
||||
const suffix = skill?.trim().toUpperCase().replace(/[^A-Z0-9]/gu, "_");
|
||||
return suffix || undefined;
|
||||
}
|
||||
|
||||
function getPositiveNumberForSkill(
|
||||
envKey: string,
|
||||
skill: string | undefined,
|
||||
fallback: number,
|
||||
): number {
|
||||
const skillSuffix = normalizeSkillEnvSuffix(skill);
|
||||
if (skillSuffix) {
|
||||
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
|
||||
if (raw) {
|
||||
const parsed = Number(raw);
|
||||
if (Number.isFinite(parsed) && parsed > 0) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
return getPositiveNumber(envKey, fallback);
|
||||
}
|
||||
|
||||
function getRatioNumberForSkill(
|
||||
envKey: string,
|
||||
skill: string | undefined,
|
||||
fallback: number,
|
||||
): number {
|
||||
const skillSuffix = normalizeSkillEnvSuffix(skill);
|
||||
if (skillSuffix) {
|
||||
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
|
||||
if (raw) {
|
||||
const parsed = Number(raw);
|
||||
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
return getRatioNumber(envKey, fallback);
|
||||
}
|
||||
|
||||
function getBoolean(envKey: string, fallback: boolean): boolean {
|
||||
const raw = process.env[envKey]?.trim().toLowerCase();
|
||||
if (!raw) return fallback;
|
||||
@@ -539,22 +580,42 @@ export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
|
||||
}
|
||||
}
|
||||
|
||||
export function getAlertThresholdP95LatencyMs(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
|
||||
export function getAlertThresholdP95LatencyMs(skill?: string): number {
|
||||
return getPositiveNumberForSkill(
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
||||
skill,
|
||||
3_000,
|
||||
);
|
||||
}
|
||||
|
||||
export function getAlertThresholdEvalScoreDropPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
|
||||
export function getAlertThresholdEvalScoreDropPct(skill?: string): number {
|
||||
return getPositiveNumberForSkill(
|
||||
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT",
|
||||
skill,
|
||||
10,
|
||||
);
|
||||
}
|
||||
|
||||
export function getAlertThresholdLlmErrorRate(): number {
|
||||
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
|
||||
export function getAlertThresholdLlmErrorRate(skill?: string): number {
|
||||
return getRatioNumberForSkill(
|
||||
"ALERT_THRESHOLD_LLM_ERROR_RATE",
|
||||
skill,
|
||||
0.1,
|
||||
);
|
||||
}
|
||||
|
||||
export function getAlertThresholdDriftPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15);
|
||||
export function getAlertThresholdDriftPct(skill?: string): number {
|
||||
return getPositiveNumberForSkill("ALERT_THRESHOLD_DRIFT_PCT", skill, 15);
|
||||
}
|
||||
|
||||
export function getAlertRateLimitSec(): number {
|
||||
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
|
||||
}
|
||||
|
||||
export function getSlackAlertWebhookUrl(): string | undefined {
|
||||
return process.env["SLACK_ALERT_WEBHOOK_URL"]?.trim() || undefined;
|
||||
}
|
||||
|
||||
export function getSlackWebhookTimeoutMs(): number {
|
||||
return getPositiveNumber("SLACK_WEBHOOK_TIMEOUT_MS", 3_000);
|
||||
}
|
||||
Reference in new issue
Block a user