Add Slack alerts and skill-aware monitoring
This commit is contained in:
1 parent
b63985694a
commit
f1b30d716d
19 files changed
+1288
-112
No files matched your search
@@ -0,0 +1,102 @@
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import {
|
||||
getSlackAlertWebhookUrl,
|
||||
getSlackWebhookTimeoutMs,
|
||||
} from "../config.js";
|
||||
import { getLogger } from "./logger.js";
|
||||
|
||||
export interface SlackTextObject {
|
||||
text: string;
|
||||
type: "mrkdwn" | "plain_text";
|
||||
}
|
||||
|
||||
export interface SlackBlock {
|
||||
fields?: SlackTextObject[] | undefined;
|
||||
text?: SlackTextObject | undefined;
|
||||
type: "section";
|
||||
}
|
||||
|
||||
export interface SlackMessage {
|
||||
blocks: SlackBlock[];
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface SlackNotifier {
|
||||
notify(payload: AlertPayload): Promise<boolean>;
|
||||
}
|
||||
|
||||
const SEVERITY_EMOJI: Record<AlertPayload["severity"], string> = {
|
||||
critical: ":rotating_light:",
|
||||
info: ":information_source:",
|
||||
warn: ":warning:",
|
||||
};
|
||||
|
||||
function mrkdwn(text: string): SlackTextObject {
|
||||
return { text, type: "mrkdwn" };
|
||||
}
|
||||
|
||||
export function formatSlackMessage(payload: AlertPayload): SlackMessage {
|
||||
const blocks: SlackBlock[] = [
|
||||
{
|
||||
text: mrkdwn(
|
||||
`*${payload.severity.toUpperCase()}* - \`${payload.kind}\``,
|
||||
),
|
||||
type: "section",
|
||||
},
|
||||
{
|
||||
fields: [
|
||||
mrkdwn(`*Metric*\n${payload.metric}`),
|
||||
mrkdwn(`*Value / Threshold*\n${payload.value} / ${payload.threshold}`),
|
||||
mrkdwn(`*Window*\n${payload.window}`),
|
||||
mrkdwn(`*Skill*\n${payload.skill ?? "global"}`),
|
||||
],
|
||||
type: "section",
|
||||
},
|
||||
];
|
||||
|
||||
if (payload.suggestedAction?.trim()) {
|
||||
blocks.push({
|
||||
text: mrkdwn(`*Suggested action*\n${payload.suggestedAction}`),
|
||||
type: "section",
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
blocks,
|
||||
text: `${SEVERITY_EMOJI[payload.severity]} [${payload.kind}] ${payload.metric}`,
|
||||
};
|
||||
}
|
||||
|
||||
export class SlackWebhookNotifier implements SlackNotifier {
|
||||
constructor(
|
||||
private readonly webhookUrl: string,
|
||||
private readonly fetchImpl: typeof fetch = fetch,
|
||||
) {}
|
||||
|
||||
async notify(payload: AlertPayload): Promise<boolean> {
|
||||
try {
|
||||
const response = await this.fetchImpl(this.webhookUrl, {
|
||||
body: JSON.stringify(formatSlackMessage(payload)),
|
||||
headers: { "Content-Type": "application/json" },
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(getSlackWebhookTimeoutMs()),
|
||||
});
|
||||
return response.ok;
|
||||
} catch (error) {
|
||||
getLogger().warn(
|
||||
{
|
||||
err: error,
|
||||
kind: payload.kind,
|
||||
skill: payload.skill,
|
||||
},
|
||||
"[Slack] Notification failed",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createSlackNotifierFromEnv(): SlackNotifier | undefined {
|
||||
const webhookUrl = getSlackAlertWebhookUrl();
|
||||
return webhookUrl ? new SlackWebhookNotifier(webhookUrl) : undefined;
|
||||
}
|
||||
Reference in new issue
Block a user