Add Slack alerts and skill-aware monitoring

This commit is contained in:
dahyeon committed 2026-05-18 19:11:05 -07:00
1 parent b63985694a
commit f1b30d716d
19 files changed
+1288 -112

No files matched your search

+12 -3
View File
@@ -174,11 +174,15 @@ export class Agent {
metrics.handoffTotal.inc({
from: event.from,
reason: event.reason,
skill: event.from,
to: event.to,
});
},
onHandoffLoop: (event) => {
metrics.handoffLoopTotal.inc({ from: event.from });
metrics.handoffLoopTotal.inc({
from: event.from,
skill: event.from,
});
},
onValidatorFail: (event) => {
metrics.validatorFailTotal.inc({
@@ -206,8 +210,13 @@ export class Agent {
}
await this.natsClient.respond(msg, response);
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
metrics.handoffCount.observe(
Math.max(0, budget.snapshot().handoffs),
{ skill: skill.name },
);
metrics.timeToResolutionMs.observe(Date.now() - startedAt, {
skill: skill.name,
});
rootSpan.setAttribute(
"agent.source",
response.text.trim() ? response.source : "empty",
+1
View File
@@ -9,6 +9,7 @@ export const KNOWN_SECRET_KEYS = [
"GARNET_PASSWORD",
"MONGODB_CONNECTION_STRING",
"MONGODB_PROJECT_ID",
"SLACK_ALERT_WEBHOOK_URL",
] as const;
export type KnownSecretKey = (typeof KNOWN_SECRET_KEYS)[number];
+1
View File
@@ -17,6 +17,7 @@ export interface AlertPayload {
| "supervisor-handoff";
metric: string;
severity: "info" | "warn" | "critical";
skill?: string | undefined;
suggestedAction?: string | undefined;
threshold: number;
timestamp: string;
+62
View File
@@ -0,0 +1,62 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
getAlertThresholdDriftPct,
getAlertThresholdEvalScoreDropPct,
getAlertThresholdLlmErrorRate,
getAlertThresholdP95LatencyMs,
getSlackAlertWebhookUrl,
getSlackWebhookTimeoutMs,
} from "./config.js";
const ENV_KEYS = [
"ALERT_THRESHOLD_DRIFT_PCT",
"ALERT_THRESHOLD_DRIFT_PCT_ONBOARDING",
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT",
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT_MODERATION",
"ALERT_THRESHOLD_LLM_ERROR_RATE",
"ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING",
"ALERT_THRESHOLD_P95_LATENCY_MS",
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
"SLACK_ALERT_WEBHOOK_URL",
"SLACK_WEBHOOK_TIMEOUT_MS",
] as const;
describe("config", () => {
afterEach(() => {
for (const key of ENV_KEYS) {
vi.stubEnv(key, undefined);
}
vi.unstubAllEnvs();
});
it("prefers skill-specific alert thresholds", () => {
vi.stubEnv("ALERT_THRESHOLD_P95_LATENCY_MS", "3000");
vi.stubEnv("ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING", "5000");
vi.stubEnv("ALERT_THRESHOLD_LLM_ERROR_RATE", "0.1");
vi.stubEnv("ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING", "0.25");
expect(getAlertThresholdP95LatencyMs("onboarding")).toBe(5000);
expect(getAlertThresholdP95LatencyMs("moderation")).toBe(3000);
expect(getAlertThresholdLlmErrorRate("onboarding")).toBe(0.25);
});
it("normalizes skill names and falls back to global values", () => {
vi.stubEnv("ALERT_THRESHOLD_DRIFT_PCT", "15");
vi.stubEnv("ALERT_THRESHOLD_DRIFT_PCT_HIGH_RISK", "7");
vi.stubEnv("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", "10");
vi.stubEnv("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT_MODERATION", "6");
expect(getAlertThresholdDriftPct("high-risk")).toBe(7);
expect(getAlertThresholdDriftPct("onboarding")).toBe(15);
expect(getAlertThresholdEvalScoreDropPct("moderation")).toBe(6);
});
it("reads Slack webhook configuration", () => {
vi.stubEnv("SLACK_ALERT_WEBHOOK_URL", " https://hooks.slack.test/x ");
vi.stubEnv("SLACK_WEBHOOK_TIMEOUT_MS", "1234");
expect(getSlackAlertWebhookUrl()).toBe("https://hooks.slack.test/x");
expect(getSlackWebhookTimeoutMs()).toBe(1234);
});
});
+69 -8
View File
@@ -29,6 +29,47 @@ function getRatioNumber(
: fallback;
}
function normalizeSkillEnvSuffix(skill: string | undefined): string | undefined {
const suffix = skill?.trim().toUpperCase().replace(/[^A-Z0-9]/gu, "_");
return suffix || undefined;
}
function getPositiveNumberForSkill(
envKey: string,
skill: string | undefined,
fallback: number,
): number {
const skillSuffix = normalizeSkillEnvSuffix(skill);
if (skillSuffix) {
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
if (raw) {
const parsed = Number(raw);
if (Number.isFinite(parsed) && parsed > 0) {
return parsed;
}
}
}
return getPositiveNumber(envKey, fallback);
}
function getRatioNumberForSkill(
envKey: string,
skill: string | undefined,
fallback: number,
): number {
const skillSuffix = normalizeSkillEnvSuffix(skill);
if (skillSuffix) {
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
if (raw) {
const parsed = Number(raw);
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
return parsed;
}
}
}
return getRatioNumber(envKey, fallback);
}
function getBoolean(envKey: string, fallback: boolean): boolean {
const raw = process.env[envKey]?.trim().toLowerCase();
if (!raw) return fallback;
@@ -539,22 +580,42 @@ export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
}
}
export function getAlertThresholdP95LatencyMs(): number {
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
export function getAlertThresholdP95LatencyMs(skill?: string): number {
return getPositiveNumberForSkill(
"ALERT_THRESHOLD_P95_LATENCY_MS",
skill,
3_000,
);
}
export function getAlertThresholdEvalScoreDropPct(): number {
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
export function getAlertThresholdEvalScoreDropPct(skill?: string): number {
return getPositiveNumberForSkill(
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT",
skill,
10,
);
}
export function getAlertThresholdLlmErrorRate(): number {
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
export function getAlertThresholdLlmErrorRate(skill?: string): number {
return getRatioNumberForSkill(
"ALERT_THRESHOLD_LLM_ERROR_RATE",
skill,
0.1,
);
}
export function getAlertThresholdDriftPct(): number {
return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15);
export function getAlertThresholdDriftPct(skill?: string): number {
return getPositiveNumberForSkill("ALERT_THRESHOLD_DRIFT_PCT", skill, 15);
}
export function getAlertRateLimitSec(): number {
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
}
export function getSlackAlertWebhookUrl(): string | undefined {
return process.env["SLACK_ALERT_WEBHOOK_URL"]?.trim() || undefined;
}
export function getSlackWebhookTimeoutMs(): number {
return getPositiveNumber("SLACK_WEBHOOK_TIMEOUT_MS", 3_000);
}
+62 -1
View File
@@ -1,9 +1,11 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import type { AlertPayload } from "@common/nats-subjects";
import {
calculateEvalScoreDropPct,
calculateLlmErrorRate,
calculatePercentile,
ShadowWorker,
} from "./shadow-worker.js";
describe("calculatePercentile", () => {
@@ -40,14 +42,73 @@ describe("calculateLlmErrorRate", () => {
llmOverall: null,
recordedAt: 1,
responseLatencyMs: 100,
skill: "onboarding",
},
{
llmJudgeErrored: false,
llmOverall: 4.2,
recordedAt: 2,
responseLatencyMs: 200,
skill: "onboarding",
},
]),
).toBe(0.5);
});
});
describe("ShadowWorker alerts", () => {
it("emits latency alerts with the observation skill", async () => {
const sendAlert = vi.fn().mockResolvedValue(true);
const worker = Object.assign(Object.create(ShadowWorker.prototype), {
alertSender: { sendAlert },
recentEvalObservations: [
{
llmJudgeErrored: false,
llmOverall: 4,
recordedAt: Date.now(),
responseLatencyMs: 4500,
skill: "onboarding",
},
{
llmJudgeErrored: false,
llmOverall: 4,
recordedAt: Date.now(),
responseLatencyMs: 4600,
skill: "onboarding",
},
{
llmJudgeErrored: false,
llmOverall: 4,
recordedAt: Date.now(),
responseLatencyMs: 4700,
skill: "onboarding",
},
{
llmJudgeErrored: false,
llmOverall: 4,
recordedAt: Date.now(),
responseLatencyMs: 4800,
skill: "onboarding",
},
{
llmJudgeErrored: false,
llmOverall: 4,
recordedAt: Date.now(),
responseLatencyMs: 4900,
skill: "onboarding",
},
],
}) as {
checkLatencyAlert: (now: number) => Promise<void>;
};
await worker.checkLatencyAlert(Date.now());
expect(sendAlert).toHaveBeenCalledWith(
expect.objectContaining<Partial<AlertPayload>>({
kind: "latency",
skill: "onboarding",
}),
);
});
});
+105 -64
View File
@@ -112,6 +112,7 @@ interface EvalObservation {
llmOverall: number | null;
recordedAt: number;
responseLatencyMs: number | null;
skill: string;
}
function delay(ms: number): Promise<void> {
@@ -331,6 +332,18 @@ export function calculateLlmErrorRate(
return errorCount / observations.length;
}
function groupObservationsBySkill(
observations: EvalObservation[],
): Map<string, EvalObservation[]> {
const groups = new Map<string, EvalObservation[]>();
for (const observation of observations) {
const values = groups.get(observation.skill) ?? [];
values.push(observation);
groups.set(observation.skill, values);
}
return groups;
}
export class ShadowWorker {
private archiveFailureCount = 0;
private lastAlertCheckAt = 0;
@@ -586,6 +599,7 @@ export class ShadowWorker {
llmOverall,
recordedAt: Date.now(),
responseLatencyMs: job.latencyMs ?? null,
skill: job.skill,
});
const resultFields: Record<string, string> = {
@@ -938,17 +952,22 @@ export class ShadowWorker {
return;
}
const threshold = getAlertThresholdDriftPct();
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
const significantReports = reports.filter((report) => report.significant);
const significantReports = reports.filter((report) => {
const threshold = getAlertThresholdDriftPct(report.slice.skill);
return report.recent24hCount >= 30 && report.dropPct >= threshold;
});
for (const report of significantReports) {
const slice = sliceKeyToString(report.slice);
const skill = report.slice.skill ?? "onboarding";
const threshold = getAlertThresholdDriftPct(skill);
await this.alertSender.sendAlert(
{
kind: "eval-drift",
metric: "shadow_eval.drift.llm_overall_drop_pct",
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
threshold,
timestamp: new Date(now).toISOString(),
@@ -1059,29 +1078,40 @@ export class ShadowWorker {
}
private async checkLatencyAlert(now: number): Promise<void> {
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
);
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
if (!this.alertSender) {
return;
}
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
const threshold = getAlertThresholdP95LatencyMs();
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
return;
}
for (const [skill, observations] of groupObservationsBySkill(
this.recentEvalObservations,
)) {
const latencySamples = observations.flatMap((observation) =>
observation.responseLatencyMs === null
? []
: [observation.responseLatencyMs],
);
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS) {
continue;
}
await this.alertSender.sendAlert({
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: p95LatencyMs,
window: ALERT_WINDOW_LABEL,
});
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
const threshold = getAlertThresholdP95LatencyMs(skill);
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
continue;
}
await this.alertSender.sendAlert({
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: p95LatencyMs,
window: ALERT_WINDOW_LABEL,
});
}
}
private async checkEvalScoreDropAlert(now: number): Promise<void> {
@@ -1089,59 +1119,70 @@ export class ShadowWorker {
return;
}
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
observation.llmOverall === null ? [] : [observation.llmOverall],
);
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
return;
}
const currentAverage =
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
const baselineAverage = await this.loadBaselineOverallScore();
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
const threshold = getAlertThresholdEvalScoreDropPct();
for (const [skill, observations] of groupObservationsBySkill(
this.recentEvalObservations,
)) {
const scoreSamples = observations.flatMap((observation) =>
observation.llmOverall === null ? [] : [observation.llmOverall],
);
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
continue;
}
if (dropPct === null || dropPct <= threshold) {
return;
const currentAverage =
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
const threshold = getAlertThresholdEvalScoreDropPct(skill);
if (dropPct === null || dropPct <= threshold) {
continue;
}
await this.alertSender.sendAlert({
kind: "eval-score-drop",
metric: "shadow_eval.llm_overall_drop_pct",
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(dropPct.toFixed(3)),
window: ALERT_WINDOW_LABEL,
});
}
await this.alertSender.sendAlert({
kind: "eval-score-drop",
metric: "shadow_eval.llm_overall_drop_pct",
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(dropPct.toFixed(3)),
window: ALERT_WINDOW_LABEL,
});
}
private async checkLlmErrorRateAlert(now: number): Promise<void> {
if (
!this.alertSender ||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
) {
if (!this.alertSender) {
return;
}
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
const threshold = getAlertThresholdLlmErrorRate();
if (errorRate === null || errorRate <= threshold) {
return;
}
for (const [skill, observations] of groupObservationsBySkill(
this.recentEvalObservations,
)) {
if (observations.length < MIN_ERROR_RATE_OBSERVATIONS) {
continue;
}
await this.alertSender.sendAlert({
kind: "llm-error-rate",
metric: "shadow_eval.llm_error_rate",
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(errorRate.toFixed(4)),
window: ALERT_WINDOW_LABEL,
});
const errorRate = calculateLlmErrorRate(observations);
const threshold = getAlertThresholdLlmErrorRate(skill);
if (errorRate === null || errorRate <= threshold) {
continue;
}
await this.alertSender.sendAlert({
kind: "llm-error-rate",
metric: "shadow_eval.llm_error_rate",
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
skill,
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
threshold,
timestamp: new Date(now).toISOString(),
value: Number(errorRate.toFixed(4)),
window: ALERT_WINDOW_LABEL,
});
}
}
private async checkOpenCircuitAlerts(now: number): Promise<void> {
+5 -1
View File
@@ -7,7 +7,7 @@ import type {
LlmEndpointConfig,
ProviderName,
} from "./types.js";
import { getRequestLogger } from "@obs/context";
import { getContext, getRequestLogger } from "@obs/context";
import { withLlmSpan } from "@obs/spans";
import {
getLlmHealthTimeoutMs,
@@ -219,6 +219,10 @@ export class LlmClient {
payload.usage?.prompt_tokens,
payload.usage?.completion_tokens,
),
{
model: request.model,
skill: getContext()?.skill,
},
);
span.setAttributes({
+37 -1
View File
@@ -8,6 +8,7 @@ const payload: AlertPayload = {
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: "warn",
skill: "onboarding",
threshold: 3000,
timestamp: "2026-04-17T00:00:00.000Z",
value: 4100,
@@ -23,7 +24,11 @@ describe("AlertService", () => {
const result = await service.sendAlert(payload);
expect(result).toBe(true);
expect(setStringIfAbsent).toHaveBeenCalledTimes(1);
expect(setStringIfAbsent).toHaveBeenCalledWith(
"lolympus-agent:alert:dedupe:latency:onboarding",
payload.timestamp,
expect.any(Number),
);
expect(publish).toHaveBeenCalledWith(AGENT_ALERT_SUBJECT, payload);
});
@@ -37,4 +42,35 @@ describe("AlertService", () => {
expect(result).toBe(false);
expect(publish).not.toHaveBeenCalled();
});
it("keeps dedupe global when payload has no skill", async () => {
const setStringIfAbsent = vi.fn().mockResolvedValue(true);
const publish = vi.fn().mockResolvedValue(undefined);
const service = new AlertService({ setStringIfAbsent }, { publish });
await service.sendAlert({ ...payload, skill: undefined });
expect(setStringIfAbsent).toHaveBeenCalledWith(
"lolympus-agent:alert:dedupe:latency",
payload.timestamp,
expect.any(Number),
);
});
it("notifies Slack without changing publish result", async () => {
const setStringIfAbsent = vi.fn().mockResolvedValue(true);
const publish = vi.fn().mockResolvedValue(undefined);
const notify = vi.fn().mockResolvedValue(false);
const service = new AlertService(
{ setStringIfAbsent },
{ publish },
{ notify },
);
const result = await service.sendAlert(payload);
expect(result).toBe(true);
expect(publish).toHaveBeenCalledWith(AGENT_ALERT_SUBJECT, payload);
expect(notify).toHaveBeenCalledWith(payload);
});
});
+41 -3
View File
@@ -3,6 +3,7 @@ import { AGENT_ALERT_SUBJECT, type AlertPayload } from "@common/nats-subjects";
import type { NatsClient } from "@common/nats";
import { getAlertRateLimitSec } from "../config.js";
import { getLogger } from "./logger.js";
import type { SlackNotifier } from "./slack-notifier.js";
export interface AlertSender {
sendAlert(
@@ -13,8 +14,11 @@ export interface AlertSender {
): Promise<boolean>;
}
function getAlertDedupeKey(kind: AlertPayload["kind"]): string {
return `lolympus-agent:alert:dedupe:${kind}`;
function getAlertDedupeKey(
kind: AlertPayload["kind"],
skill?: string,
): string {
return `lolympus-agent:alert:dedupe:${kind}${skill ? `:${skill}` : ""}`;
}
export class AlertService implements AlertSender {
@@ -23,6 +27,7 @@ export class AlertService implements AlertSender {
constructor(
private readonly garnetClient: Pick<GarnetClient, "setStringIfAbsent">,
private readonly natsClient: Pick<NatsClient, "publish">,
private readonly slackNotifier?: SlackNotifier | undefined,
) {}
async sendAlert(
@@ -31,7 +36,8 @@ export class AlertService implements AlertSender {
dedupeKey?: string | undefined;
},
): Promise<boolean> {
const dedupeKey = options?.dedupeKey ?? getAlertDedupeKey(payload.kind);
const dedupeKey =
options?.dedupeKey ?? getAlertDedupeKey(payload.kind, payload.skill);
const claimed = await this.garnetClient.setStringIfAbsent(
dedupeKey,
payload.timestamp,
@@ -43,6 +49,7 @@ export class AlertService implements AlertSender {
{
dedupeKey,
kind: payload.kind,
skill: payload.skill,
},
"[Alert] Suppressed by rate limit",
);
@@ -55,12 +62,43 @@ export class AlertService implements AlertSender {
kind: payload.kind,
metric: payload.metric,
severity: payload.severity,
skill: payload.skill,
threshold: payload.threshold,
value: payload.value,
window: payload.window,
},
"[Alert] Published",
);
this.notifySlack(payload);
return true;
}
private notifySlack(payload: AlertPayload): void {
if (!this.slackNotifier) {
return;
}
void this.slackNotifier.notify(payload)
.then((ok) => {
if (!ok) {
this.logger.warn(
{
kind: payload.kind,
skill: payload.skill,
},
"[Alert] Slack notify failed",
);
}
})
.catch((error) => {
this.logger.warn(
{
err: error,
kind: payload.kind,
skill: payload.skill,
},
"[Alert] Slack notify failed",
);
});
}
}
+25
View File
@@ -0,0 +1,25 @@
import { describe, expect, it, vi } from "vitest";
import { handleMetricsRequest } from "./metrics-server.js";
describe("metrics server", () => {
it("serves metrics and health checks", () => {
const writeHead = vi.fn();
const end = vi.fn();
handleMetricsRequest({ url: "/healthz" }, { end, writeHead });
expect(writeHead).toHaveBeenCalledWith(200, { "Content-Type": "text/plain" });
expect(end).toHaveBeenCalledWith("ok");
writeHead.mockClear();
end.mockClear();
handleMetricsRequest({ url: "/metrics" }, { end, writeHead });
expect(writeHead).toHaveBeenCalledWith(200, {
"Content-Type": "text/plain; version=0.0.4",
});
expect(String(end.mock.calls[0]?.[0])).toContain(
"# HELP agent_handoff_total",
);
});
});
+29 -13
View File
@@ -1,4 +1,9 @@
import { createServer, type Server } from "node:http";
import {
createServer,
type IncomingMessage,
type Server,
type ServerResponse,
} from "node:http";
import { getMetricsPort } from "../config.js";
import { getLogger } from "./logger.js";
@@ -6,23 +11,34 @@ import { renderMetrics } from "./metrics.js";
let server: Server | null = null;
export function handleMetricsRequest(
request: Pick<IncomingMessage, "url">,
response: Pick<ServerResponse, "end" | "writeHead">,
): void {
if (request.url === "/healthz") {
response.writeHead(200, { "Content-Type": "text/plain" });
response.end("ok");
return;
}
if (request.url !== "/metrics") {
response.writeHead(404);
response.end("not found");
return;
}
response.writeHead(200, {
"Content-Type": "text/plain; version=0.0.4",
});
response.end(renderMetrics());
}
export function startMetricsServer(port = getMetricsPort()): Server {
if (server) {
return server;
}
server = createServer((request, response) => {
if (request.url !== "/metrics") {
response.writeHead(404);
response.end("not found");
return;
}
response.writeHead(200, {
"Content-Type": "text/plain; version=0.0.4",
});
response.end(renderMetrics());
});
server = createServer(handleMetricsRequest);
server.listen(port, () => {
getLogger().info({ port }, "✓ Metrics server listening");
+6 -3
View File
@@ -26,15 +26,18 @@ describe("metrics", () => {
metrics.handoffTotal.inc({
from: "onboarding",
reason: "out-of-scope",
skill: "onboarding",
to: "moderation",
});
metrics.handoffCount.observe(2);
metrics.handoffCount.observe(2, { skill: "onboarding" });
const body = renderMetrics();
expect(body).toContain(
'agent_handoff_total{from="onboarding",reason="out-of-scope",to="moderation"} 1',
'agent_handoff_total{from="onboarding",reason="out-of-scope",skill="onboarding",to="moderation"} 1',
);
expect(body).toContain(
'agent_handoff_count_bucket{skill="onboarding",le="2"} 1',
);
expect(body).toContain('agent_handoff_count_bucket{le="2"} 1');
});
});
+33 -12
View File
@@ -52,7 +52,7 @@ class CounterMetric {
}
class HistogramMetric {
private readonly values: number[] = [];
private readonly samples = new Map<string, { labels: Labels; values: number[] }>();
constructor(
readonly name: string,
@@ -60,10 +60,15 @@ class HistogramMetric {
private readonly buckets: number[],
) {}
observe(value: number): void {
if (Number.isFinite(value)) {
this.values.push(value);
observe(value: number, labels: Labels = {}): void {
if (!Number.isFinite(value)) {
return;
}
const key = labelKey(labels);
const sample = this.samples.get(key) ?? { labels, values: [] };
sample.values.push(value);
this.samples.set(key, sample);
}
render(): string[] {
@@ -71,15 +76,31 @@ class HistogramMetric {
`# HELP ${this.name} ${this.help}`,
`# TYPE ${this.name} histogram`,
];
for (const bucket of this.buckets) {
const count = this.values.filter((value) => value <= bucket).length;
lines.push(`${this.name}_bucket{le="${bucket}"} ${count}`);
for (const sample of this.samples.values()) {
for (const bucket of this.buckets) {
const count = sample.values.filter((value) => value <= bucket).length;
lines.push(
`${this.name}_bucket${formatLabels({
...sample.labels,
le: bucket,
})} ${count}`,
);
}
lines.push(
`${this.name}_bucket${formatLabels({
...sample.labels,
le: "+Inf",
})} ${sample.values.length}`,
);
lines.push(
`${this.name}_sum${formatLabels(sample.labels)} ${sample.values.reduce(
(sum, value) => sum + value,
0,
)}`,
);
lines.push(`${this.name}_count${formatLabels(sample.labels)} ${sample.values.length}`);
}
lines.push(`${this.name}_bucket{le="+Inf"} ${this.values.length}`);
lines.push(
`${this.name}_sum ${this.values.reduce((sum, value) => sum + value, 0)}`,
);
lines.push(`${this.name}_count ${this.values.length}`);
return lines;
}
}
+89
View File
@@ -0,0 +1,89 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { AlertPayload } from "@common/nats-subjects";
import {
formatSlackMessage,
SlackWebhookNotifier,
} from "./slack-notifier.js";
const payload: AlertPayload = {
kind: "latency",
metric: "shadow_eval.p95_latency_ms",
severity: "critical",
skill: "onboarding",
suggestedAction: "Check endpoint latency.",
threshold: 3000,
timestamp: "2026-04-17T00:00:00.000Z",
value: 4500,
window: "5m",
};
describe("formatSlackMessage", () => {
it("formats severity, metric, skill, and action", () => {
const message = formatSlackMessage(payload);
expect(message.text).toBe(
":rotating_light: [latency] shadow_eval.p95_latency_ms",
);
expect(JSON.stringify(message.blocks)).toContain("*Skill*\\nonboarding");
expect(JSON.stringify(message.blocks)).toContain("Check endpoint latency.");
});
});
describe("SlackWebhookNotifier", () => {
afterEach(() => {
vi.unstubAllEnvs();
});
it("posts the formatted Slack message", async () => {
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(new Response("ok"));
const notifier = new SlackWebhookNotifier(
"https://hooks.slack.test/services/test",
fetchImpl,
);
await expect(notifier.notify(payload)).resolves.toBe(true);
const call = fetchImpl.mock.calls[0];
expect(call?.[0]).toBe("https://hooks.slack.test/services/test");
expect(call?.[1]?.body).toBe(JSON.stringify(formatSlackMessage(payload)));
expect(call?.[1]?.headers).toEqual({ "Content-Type": "application/json" });
expect(call?.[1]?.method).toBe("POST");
expect(call?.[1]?.signal).toBeInstanceOf(AbortSignal);
});
it("returns false for non-2xx responses", async () => {
const fetchImpl = vi
.fn<typeof fetch>()
.mockResolvedValue(new Response("rate limited", { status: 429 }));
const notifier = new SlackWebhookNotifier(
"https://hooks.slack.test/services/test",
fetchImpl,
);
await expect(notifier.notify(payload)).resolves.toBe(false);
});
it("returns false when fetch rejects", async () => {
const fetchImpl = vi.fn<typeof fetch>().mockRejectedValue(new Error("timeout"));
const notifier = new SlackWebhookNotifier(
"https://hooks.slack.test/services/test",
fetchImpl,
);
await expect(notifier.notify(payload)).resolves.toBe(false);
});
it("uses the configured timeout signal", async () => {
vi.stubEnv("SLACK_WEBHOOK_TIMEOUT_MS", "1234");
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(new Response("ok"));
const notifier = new SlackWebhookNotifier(
"https://hooks.slack.test/services/test",
fetchImpl,
);
await notifier.notify(payload);
expect(fetchImpl.mock.calls[0]?.[1]?.signal).toBeInstanceOf(AbortSignal);
});
});
+102
View File
@@ -0,0 +1,102 @@
import type { AlertPayload } from "@common/nats-subjects";
import {
getSlackAlertWebhookUrl,
getSlackWebhookTimeoutMs,
} from "../config.js";
import { getLogger } from "./logger.js";
export interface SlackTextObject {
text: string;
type: "mrkdwn" | "plain_text";
}
export interface SlackBlock {
fields?: SlackTextObject[] | undefined;
text?: SlackTextObject | undefined;
type: "section";
}
export interface SlackMessage {
blocks: SlackBlock[];
text: string;
}
export interface SlackNotifier {
notify(payload: AlertPayload): Promise<boolean>;
}
const SEVERITY_EMOJI: Record<AlertPayload["severity"], string> = {
critical: ":rotating_light:",
info: ":information_source:",
warn: ":warning:",
};
function mrkdwn(text: string): SlackTextObject {
return { text, type: "mrkdwn" };
}
export function formatSlackMessage(payload: AlertPayload): SlackMessage {
const blocks: SlackBlock[] = [
{
text: mrkdwn(
`*${payload.severity.toUpperCase()}* - \`${payload.kind}\``,
),
type: "section",
},
{
fields: [
mrkdwn(`*Metric*\n${payload.metric}`),
mrkdwn(`*Value / Threshold*\n${payload.value} / ${payload.threshold}`),
mrkdwn(`*Window*\n${payload.window}`),
mrkdwn(`*Skill*\n${payload.skill ?? "global"}`),
],
type: "section",
},
];
if (payload.suggestedAction?.trim()) {
blocks.push({
text: mrkdwn(`*Suggested action*\n${payload.suggestedAction}`),
type: "section",
});
}
return {
blocks,
text: `${SEVERITY_EMOJI[payload.severity]} [${payload.kind}] ${payload.metric}`,
};
}
export class SlackWebhookNotifier implements SlackNotifier {
constructor(
private readonly webhookUrl: string,
private readonly fetchImpl: typeof fetch = fetch,
) {}
async notify(payload: AlertPayload): Promise<boolean> {
try {
const response = await this.fetchImpl(this.webhookUrl, {
body: JSON.stringify(formatSlackMessage(payload)),
headers: { "Content-Type": "application/json" },
method: "POST",
signal: AbortSignal.timeout(getSlackWebhookTimeoutMs()),
});
return response.ok;
} catch (error) {
getLogger().warn(
{
err: error,
kind: payload.kind,
skill: payload.skill,
},
"[Slack] Notification failed",
);
return false;
}
}
}
export function createSlackNotifierFromEnv(): SlackNotifier | undefined {
const webhookUrl = getSlackAlertWebhookUrl();
return webhookUrl ? new SlackWebhookNotifier(webhookUrl) : undefined;
}
+8 -2
View File
@@ -10,6 +10,7 @@ import type { NatsClient } from "@common/nats";
import type { LlmClient } from "@llm/llm-client";
import { AlertService } from "@obs/alert";
import { metrics } from "@obs/metrics";
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
import type { Skill } from "@skills/skill";
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
import { getLogger } from "@obs/logger";
@@ -52,7 +53,11 @@ export class SupervisorSkill implements Skill {
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
private readonly skillRegistry?: SkillRegistry | undefined,
) {
this.alertService = new AlertService(garnetClient, natsClient);
this.alertService = new AlertService(
garnetClient,
natsClient,
createSlackNotifierFromEnv(),
);
}
canHandle(request: AgentRequest): boolean {
@@ -83,13 +88,14 @@ export class SupervisorSkill implements Skill {
target: target ?? "human-review",
userId: request.userId,
});
metrics.humanOverrideTotal.inc({ kind: reason });
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
await this.alertService.sendAlert(
{
kind: "supervisor-handoff",
metric: "agent.supervisor_handoff",
severity: "warn",
skill: this.name,
suggestedAction: "Review the supervisor handoff queue.",
threshold: 0,
timestamp: new Date().toISOString(),
+8 -1
View File
@@ -21,6 +21,9 @@ const { getRawSecret } = await import("../common/secret-manager.js");
const { ConversationStore } = await import("../conversation/index.js");
const { ShadowWorker } = await import("../eval/shadow-worker.js");
const { AlertService } = await import("../observability/alert.js");
const { createSlackNotifierFromEnv } = await import(
"../observability/slack-notifier.js"
);
const { initMongoDB, closeMongoDB } = await import("../common/mongodb.js");
function firstNonEmpty(
@@ -175,7 +178,11 @@ async function bootstrap(): Promise<void> {
providerApiKeys,
});
const conversationStore = new ConversationStore(garnetClient);
const alertService = new AlertService(garnetClient, natsClient);
const alertService = new AlertService(
garnetClient,
natsClient,
createSlackNotifierFromEnv(),
);
const mongoConnectionString = firstNonEmpty(
getRawSecret("MONGODB_CONNECTION_STRING"),
process.env["MONGODB_CONNECTION_STRING"],