Files
lolympus-agent/src/observability/alert.ts
T

67 lines
1.7 KiB
TypeScript

import type { GarnetClient } from "@common/garnet";
import { AGENT_ALERT_SUBJECT, type AlertPayload } from "@common/nats-subjects";
import type { NatsClient } from "@common/nats";
import { getAlertRateLimitSec } from "../config.js";
import { getLogger } from "./logger.js";
export interface AlertSender {
sendAlert(
payload: AlertPayload,
options?: {
dedupeKey?: string | undefined;
},
): Promise<boolean>;
}
function getAlertDedupeKey(kind: AlertPayload["kind"]): string {
return `lolympus-agent:alert:dedupe:${kind}`;
}
export class AlertService implements AlertSender {
private readonly logger = getLogger();
constructor(
private readonly garnetClient: Pick<GarnetClient, "setStringIfAbsent">,
private readonly natsClient: Pick<NatsClient, "publish">,
) {}
async sendAlert(
payload: AlertPayload,
options?: {
dedupeKey?: string | undefined;
},
): Promise<boolean> {
const dedupeKey = options?.dedupeKey ?? getAlertDedupeKey(payload.kind);
const claimed = await this.garnetClient.setStringIfAbsent(
dedupeKey,
payload.timestamp,
getAlertRateLimitSec(),
);
if (!claimed) {
this.logger.debug(
{
dedupeKey,
kind: payload.kind,
},
"[Alert] Suppressed by rate limit",
);
return false;
}
await this.natsClient.publish(AGENT_ALERT_SUBJECT, payload);
this.logger.warn(
{
kind: payload.kind,
metric: payload.metric,
severity: payload.severity,
threshold: payload.threshold,
value: payload.value,
window: payload.window,
},
"[Alert] Published",
);
return true;
}
}