Add Slack alerts and skill-aware monitoring
This commit is contained in:
1 parent
b63985694a
commit
f1b30d716d
19 files changed
+1288
-112
No files matched your search
+12
-3
@@ -174,11 +174,15 @@ export class Agent {
|
||||
metrics.handoffTotal.inc({
|
||||
from: event.from,
|
||||
reason: event.reason,
|
||||
skill: event.from,
|
||||
to: event.to,
|
||||
});
|
||||
},
|
||||
onHandoffLoop: (event) => {
|
||||
metrics.handoffLoopTotal.inc({ from: event.from });
|
||||
metrics.handoffLoopTotal.inc({
|
||||
from: event.from,
|
||||
skill: event.from,
|
||||
});
|
||||
},
|
||||
onValidatorFail: (event) => {
|
||||
metrics.validatorFailTotal.inc({
|
||||
@@ -206,8 +210,13 @@ export class Agent {
|
||||
}
|
||||
|
||||
await this.natsClient.respond(msg, response);
|
||||
metrics.handoffCount.observe(Math.max(0, budget.snapshot().handoffs));
|
||||
metrics.timeToResolutionMs.observe(Date.now() - startedAt);
|
||||
metrics.handoffCount.observe(
|
||||
Math.max(0, budget.snapshot().handoffs),
|
||||
{ skill: skill.name },
|
||||
);
|
||||
metrics.timeToResolutionMs.observe(Date.now() - startedAt, {
|
||||
skill: skill.name,
|
||||
});
|
||||
rootSpan.setAttribute(
|
||||
"agent.source",
|
||||
response.text.trim() ? response.source : "empty",
|
||||
|
||||
@@ -9,6 +9,7 @@ export const KNOWN_SECRET_KEYS = [
|
||||
"GARNET_PASSWORD",
|
||||
"MONGODB_CONNECTION_STRING",
|
||||
"MONGODB_PROJECT_ID",
|
||||
"SLACK_ALERT_WEBHOOK_URL",
|
||||
] as const;
|
||||
|
||||
export type KnownSecretKey = (typeof KNOWN_SECRET_KEYS)[number];
|
||||
@@ -17,6 +17,7 @@ export interface AlertPayload {
|
||||
| "supervisor-handoff";
|
||||
metric: string;
|
||||
severity: "info" | "warn" | "critical";
|
||||
skill?: string | undefined;
|
||||
suggestedAction?: string | undefined;
|
||||
threshold: number;
|
||||
timestamp: string;
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
getAlertThresholdDriftPct,
|
||||
getAlertThresholdEvalScoreDropPct,
|
||||
getAlertThresholdLlmErrorRate,
|
||||
getAlertThresholdP95LatencyMs,
|
||||
getSlackAlertWebhookUrl,
|
||||
getSlackWebhookTimeoutMs,
|
||||
} from "./config.js";
|
||||
|
||||
const ENV_KEYS = [
|
||||
"ALERT_THRESHOLD_DRIFT_PCT",
|
||||
"ALERT_THRESHOLD_DRIFT_PCT_ONBOARDING",
|
||||
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT",
|
||||
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT_MODERATION",
|
||||
"ALERT_THRESHOLD_LLM_ERROR_RATE",
|
||||
"ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING",
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING",
|
||||
"SLACK_ALERT_WEBHOOK_URL",
|
||||
"SLACK_WEBHOOK_TIMEOUT_MS",
|
||||
] as const;
|
||||
|
||||
describe("config", () => {
|
||||
afterEach(() => {
|
||||
for (const key of ENV_KEYS) {
|
||||
vi.stubEnv(key, undefined);
|
||||
}
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("prefers skill-specific alert thresholds", () => {
|
||||
vi.stubEnv("ALERT_THRESHOLD_P95_LATENCY_MS", "3000");
|
||||
vi.stubEnv("ALERT_THRESHOLD_P95_LATENCY_MS_ONBOARDING", "5000");
|
||||
vi.stubEnv("ALERT_THRESHOLD_LLM_ERROR_RATE", "0.1");
|
||||
vi.stubEnv("ALERT_THRESHOLD_LLM_ERROR_RATE_ONBOARDING", "0.25");
|
||||
|
||||
expect(getAlertThresholdP95LatencyMs("onboarding")).toBe(5000);
|
||||
expect(getAlertThresholdP95LatencyMs("moderation")).toBe(3000);
|
||||
expect(getAlertThresholdLlmErrorRate("onboarding")).toBe(0.25);
|
||||
});
|
||||
|
||||
it("normalizes skill names and falls back to global values", () => {
|
||||
vi.stubEnv("ALERT_THRESHOLD_DRIFT_PCT", "15");
|
||||
vi.stubEnv("ALERT_THRESHOLD_DRIFT_PCT_HIGH_RISK", "7");
|
||||
vi.stubEnv("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", "10");
|
||||
vi.stubEnv("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT_MODERATION", "6");
|
||||
|
||||
expect(getAlertThresholdDriftPct("high-risk")).toBe(7);
|
||||
expect(getAlertThresholdDriftPct("onboarding")).toBe(15);
|
||||
expect(getAlertThresholdEvalScoreDropPct("moderation")).toBe(6);
|
||||
});
|
||||
|
||||
it("reads Slack webhook configuration", () => {
|
||||
vi.stubEnv("SLACK_ALERT_WEBHOOK_URL", " https://hooks.slack.test/x ");
|
||||
vi.stubEnv("SLACK_WEBHOOK_TIMEOUT_MS", "1234");
|
||||
|
||||
expect(getSlackAlertWebhookUrl()).toBe("https://hooks.slack.test/x");
|
||||
expect(getSlackWebhookTimeoutMs()).toBe(1234);
|
||||
});
|
||||
});
|
||||
+69
-8
@@ -29,6 +29,47 @@ function getRatioNumber(
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function normalizeSkillEnvSuffix(skill: string | undefined): string | undefined {
|
||||
const suffix = skill?.trim().toUpperCase().replace(/[^A-Z0-9]/gu, "_");
|
||||
return suffix || undefined;
|
||||
}
|
||||
|
||||
function getPositiveNumberForSkill(
|
||||
envKey: string,
|
||||
skill: string | undefined,
|
||||
fallback: number,
|
||||
): number {
|
||||
const skillSuffix = normalizeSkillEnvSuffix(skill);
|
||||
if (skillSuffix) {
|
||||
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
|
||||
if (raw) {
|
||||
const parsed = Number(raw);
|
||||
if (Number.isFinite(parsed) && parsed > 0) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
return getPositiveNumber(envKey, fallback);
|
||||
}
|
||||
|
||||
function getRatioNumberForSkill(
|
||||
envKey: string,
|
||||
skill: string | undefined,
|
||||
fallback: number,
|
||||
): number {
|
||||
const skillSuffix = normalizeSkillEnvSuffix(skill);
|
||||
if (skillSuffix) {
|
||||
const raw = process.env[`${envKey}_${skillSuffix}`]?.trim();
|
||||
if (raw) {
|
||||
const parsed = Number(raw);
|
||||
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
return getRatioNumber(envKey, fallback);
|
||||
}
|
||||
|
||||
function getBoolean(envKey: string, fallback: boolean): boolean {
|
||||
const raw = process.env[envKey]?.trim().toLowerCase();
|
||||
if (!raw) return fallback;
|
||||
@@ -539,22 +580,42 @@ export function getShadowEvalSamplingRate(tier: 0 | 1 | 2): number {
|
||||
}
|
||||
}
|
||||
|
||||
export function getAlertThresholdP95LatencyMs(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_P95_LATENCY_MS", 3_000);
|
||||
export function getAlertThresholdP95LatencyMs(skill?: string): number {
|
||||
return getPositiveNumberForSkill(
|
||||
"ALERT_THRESHOLD_P95_LATENCY_MS",
|
||||
skill,
|
||||
3_000,
|
||||
);
|
||||
}
|
||||
|
||||
export function getAlertThresholdEvalScoreDropPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT", 10);
|
||||
export function getAlertThresholdEvalScoreDropPct(skill?: string): number {
|
||||
return getPositiveNumberForSkill(
|
||||
"ALERT_THRESHOLD_EVAL_SCORE_DROP_PCT",
|
||||
skill,
|
||||
10,
|
||||
);
|
||||
}
|
||||
|
||||
export function getAlertThresholdLlmErrorRate(): number {
|
||||
return getRatioNumber("ALERT_THRESHOLD_LLM_ERROR_RATE", 0.1);
|
||||
export function getAlertThresholdLlmErrorRate(skill?: string): number {
|
||||
return getRatioNumberForSkill(
|
||||
"ALERT_THRESHOLD_LLM_ERROR_RATE",
|
||||
skill,
|
||||
0.1,
|
||||
);
|
||||
}
|
||||
|
||||
export function getAlertThresholdDriftPct(): number {
|
||||
return getPositiveNumber("ALERT_THRESHOLD_DRIFT_PCT", 15);
|
||||
export function getAlertThresholdDriftPct(skill?: string): number {
|
||||
return getPositiveNumberForSkill("ALERT_THRESHOLD_DRIFT_PCT", skill, 15);
|
||||
}
|
||||
|
||||
export function getAlertRateLimitSec(): number {
|
||||
return getPositiveNumber("ALERT_RATE_LIMIT_SEC", 300);
|
||||
}
|
||||
|
||||
export function getSlackAlertWebhookUrl(): string | undefined {
|
||||
return process.env["SLACK_ALERT_WEBHOOK_URL"]?.trim() || undefined;
|
||||
}
|
||||
|
||||
export function getSlackWebhookTimeoutMs(): number {
|
||||
return getPositiveNumber("SLACK_WEBHOOK_TIMEOUT_MS", 3_000);
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import {
|
||||
calculateEvalScoreDropPct,
|
||||
calculateLlmErrorRate,
|
||||
calculatePercentile,
|
||||
ShadowWorker,
|
||||
} from "./shadow-worker.js";
|
||||
|
||||
describe("calculatePercentile", () => {
|
||||
@@ -40,14 +42,73 @@ describe("calculateLlmErrorRate", () => {
|
||||
llmOverall: null,
|
||||
recordedAt: 1,
|
||||
responseLatencyMs: 100,
|
||||
skill: "onboarding",
|
||||
},
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4.2,
|
||||
recordedAt: 2,
|
||||
responseLatencyMs: 200,
|
||||
skill: "onboarding",
|
||||
},
|
||||
]),
|
||||
).toBe(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ShadowWorker alerts", () => {
|
||||
it("emits latency alerts with the observation skill", async () => {
|
||||
const sendAlert = vi.fn().mockResolvedValue(true);
|
||||
const worker = Object.assign(Object.create(ShadowWorker.prototype), {
|
||||
alertSender: { sendAlert },
|
||||
recentEvalObservations: [
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: 4500,
|
||||
skill: "onboarding",
|
||||
},
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: 4600,
|
||||
skill: "onboarding",
|
||||
},
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: 4700,
|
||||
skill: "onboarding",
|
||||
},
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: 4800,
|
||||
skill: "onboarding",
|
||||
},
|
||||
{
|
||||
llmJudgeErrored: false,
|
||||
llmOverall: 4,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: 4900,
|
||||
skill: "onboarding",
|
||||
},
|
||||
],
|
||||
}) as {
|
||||
checkLatencyAlert: (now: number) => Promise<void>;
|
||||
};
|
||||
|
||||
await worker.checkLatencyAlert(Date.now());
|
||||
|
||||
expect(sendAlert).toHaveBeenCalledWith(
|
||||
expect.objectContaining<Partial<AlertPayload>>({
|
||||
kind: "latency",
|
||||
skill: "onboarding",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
+105
-64
@@ -112,6 +112,7 @@ interface EvalObservation {
|
||||
llmOverall: number | null;
|
||||
recordedAt: number;
|
||||
responseLatencyMs: number | null;
|
||||
skill: string;
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
@@ -331,6 +332,18 @@ export function calculateLlmErrorRate(
|
||||
return errorCount / observations.length;
|
||||
}
|
||||
|
||||
function groupObservationsBySkill(
|
||||
observations: EvalObservation[],
|
||||
): Map<string, EvalObservation[]> {
|
||||
const groups = new Map<string, EvalObservation[]>();
|
||||
for (const observation of observations) {
|
||||
const values = groups.get(observation.skill) ?? [];
|
||||
values.push(observation);
|
||||
groups.set(observation.skill, values);
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
export class ShadowWorker {
|
||||
private archiveFailureCount = 0;
|
||||
private lastAlertCheckAt = 0;
|
||||
@@ -586,6 +599,7 @@ export class ShadowWorker {
|
||||
llmOverall,
|
||||
recordedAt: Date.now(),
|
||||
responseLatencyMs: job.latencyMs ?? null,
|
||||
skill: job.skill,
|
||||
});
|
||||
|
||||
const resultFields: Record<string, string> = {
|
||||
@@ -938,17 +952,22 @@ export class ShadowWorker {
|
||||
return;
|
||||
}
|
||||
|
||||
const threshold = getAlertThresholdDriftPct();
|
||||
const reports = await computeDriftReports(this.mongoConnection, { nowMs: now });
|
||||
const significantReports = reports.filter((report) => report.significant);
|
||||
const significantReports = reports.filter((report) => {
|
||||
const threshold = getAlertThresholdDriftPct(report.slice.skill);
|
||||
return report.recent24hCount >= 30 && report.dropPct >= threshold;
|
||||
});
|
||||
|
||||
for (const report of significantReports) {
|
||||
const slice = sliceKeyToString(report.slice);
|
||||
const skill = report.slice.skill ?? "onboarding";
|
||||
const threshold = getAlertThresholdDriftPct(skill);
|
||||
await this.alertSender.sendAlert(
|
||||
{
|
||||
kind: "eval-drift",
|
||||
metric: "shadow_eval.drift.llm_overall_drop_pct",
|
||||
severity: report.dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: `${slice} slice의 최근 실패 케이스와 prompt/tool 변경 이력을 확인해 주세요.`,
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
@@ -1059,29 +1078,40 @@ export class ShadowWorker {
|
||||
}
|
||||
|
||||
private async checkLatencyAlert(now: number): Promise<void> {
|
||||
const latencySamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null ? [] : [observation.responseLatencyMs],
|
||||
);
|
||||
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS || !this.alertSender) {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
||||
const threshold = getAlertThresholdP95LatencyMs();
|
||||
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
||||
return;
|
||||
}
|
||||
for (const [skill, observations] of groupObservationsBySkill(
|
||||
this.recentEvalObservations,
|
||||
)) {
|
||||
const latencySamples = observations.flatMap((observation) =>
|
||||
observation.responseLatencyMs === null
|
||||
? []
|
||||
: [observation.responseLatencyMs],
|
||||
);
|
||||
if (latencySamples.length < MIN_LATENCY_OBSERVATIONS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: p95LatencyMs,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
const p95LatencyMs = calculatePercentile(latencySamples, 0.95);
|
||||
const threshold = getAlertThresholdP95LatencyMs(skill);
|
||||
if (p95LatencyMs === null || p95LatencyMs <= threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: p95LatencyMs >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: "최근 LLM endpoint 지연과 tool loop 호출 수를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: p95LatencyMs,
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async checkEvalScoreDropAlert(now: number): Promise<void> {
|
||||
@@ -1089,59 +1119,70 @@ export class ShadowWorker {
|
||||
return;
|
||||
}
|
||||
|
||||
const scoreSamples = this.recentEvalObservations.flatMap((observation) =>
|
||||
observation.llmOverall === null ? [] : [observation.llmOverall],
|
||||
);
|
||||
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentAverage =
|
||||
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
||||
const baselineAverage = await this.loadBaselineOverallScore();
|
||||
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
||||
const threshold = getAlertThresholdEvalScoreDropPct();
|
||||
for (const [skill, observations] of groupObservationsBySkill(
|
||||
this.recentEvalObservations,
|
||||
)) {
|
||||
const scoreSamples = observations.flatMap((observation) =>
|
||||
observation.llmOverall === null ? [] : [observation.llmOverall],
|
||||
);
|
||||
if (scoreSamples.length < MIN_SCORE_OBSERVATIONS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dropPct === null || dropPct <= threshold) {
|
||||
return;
|
||||
const currentAverage =
|
||||
scoreSamples.reduce((sum, score) => sum + score, 0) / scoreSamples.length;
|
||||
const dropPct = calculateEvalScoreDropPct(currentAverage, baselineAverage);
|
||||
const threshold = getAlertThresholdEvalScoreDropPct(skill);
|
||||
|
||||
if (dropPct === null || dropPct <= threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "eval-score-drop",
|
||||
metric: "shadow_eval.llm_overall_drop_pct",
|
||||
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(dropPct.toFixed(3)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "eval-score-drop",
|
||||
metric: "shadow_eval.llm_overall_drop_pct",
|
||||
severity: dropPct >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "최근 prompt 변경, tool 결과, fallback 비율을 함께 점검해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(dropPct.toFixed(3)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
|
||||
private async checkLlmErrorRateAlert(now: number): Promise<void> {
|
||||
if (
|
||||
!this.alertSender ||
|
||||
this.recentEvalObservations.length < MIN_ERROR_RATE_OBSERVATIONS
|
||||
) {
|
||||
if (!this.alertSender) {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorRate = calculateLlmErrorRate(this.recentEvalObservations);
|
||||
const threshold = getAlertThresholdLlmErrorRate();
|
||||
if (errorRate === null || errorRate <= threshold) {
|
||||
return;
|
||||
}
|
||||
for (const [skill, observations] of groupObservationsBySkill(
|
||||
this.recentEvalObservations,
|
||||
)) {
|
||||
if (observations.length < MIN_ERROR_RATE_OBSERVATIONS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "llm-error-rate",
|
||||
metric: "shadow_eval.llm_error_rate",
|
||||
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
||||
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(errorRate.toFixed(4)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
const errorRate = calculateLlmErrorRate(observations);
|
||||
const threshold = getAlertThresholdLlmErrorRate(skill);
|
||||
if (errorRate === null || errorRate <= threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.alertSender.sendAlert({
|
||||
kind: "llm-error-rate",
|
||||
metric: "shadow_eval.llm_error_rate",
|
||||
severity: errorRate >= threshold * 1.5 ? "critical" : "warn",
|
||||
skill,
|
||||
suggestedAction: "judge endpoint 응답 상태와 circuit breaker open 여부를 확인해 주세요.",
|
||||
threshold,
|
||||
timestamp: new Date(now).toISOString(),
|
||||
value: Number(errorRate.toFixed(4)),
|
||||
window: ALERT_WINDOW_LABEL,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async checkOpenCircuitAlerts(now: number): Promise<void> {
|
||||
|
||||
@@ -7,7 +7,7 @@ import type {
|
||||
LlmEndpointConfig,
|
||||
ProviderName,
|
||||
} from "./types.js";
|
||||
import { getRequestLogger } from "@obs/context";
|
||||
import { getContext, getRequestLogger } from "@obs/context";
|
||||
import { withLlmSpan } from "@obs/spans";
|
||||
import {
|
||||
getLlmHealthTimeoutMs,
|
||||
@@ -219,6 +219,10 @@ export class LlmClient {
|
||||
payload.usage?.prompt_tokens,
|
||||
payload.usage?.completion_tokens,
|
||||
),
|
||||
{
|
||||
model: request.model,
|
||||
skill: getContext()?.skill,
|
||||
},
|
||||
);
|
||||
|
||||
span.setAttributes({
|
||||
|
||||
@@ -8,6 +8,7 @@ const payload: AlertPayload = {
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: "warn",
|
||||
skill: "onboarding",
|
||||
threshold: 3000,
|
||||
timestamp: "2026-04-17T00:00:00.000Z",
|
||||
value: 4100,
|
||||
@@ -23,7 +24,11 @@ describe("AlertService", () => {
|
||||
const result = await service.sendAlert(payload);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(setStringIfAbsent).toHaveBeenCalledTimes(1);
|
||||
expect(setStringIfAbsent).toHaveBeenCalledWith(
|
||||
"lolympus-agent:alert:dedupe:latency:onboarding",
|
||||
payload.timestamp,
|
||||
expect.any(Number),
|
||||
);
|
||||
expect(publish).toHaveBeenCalledWith(AGENT_ALERT_SUBJECT, payload);
|
||||
});
|
||||
|
||||
@@ -37,4 +42,35 @@ describe("AlertService", () => {
|
||||
expect(result).toBe(false);
|
||||
expect(publish).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps dedupe global when payload has no skill", async () => {
|
||||
const setStringIfAbsent = vi.fn().mockResolvedValue(true);
|
||||
const publish = vi.fn().mockResolvedValue(undefined);
|
||||
const service = new AlertService({ setStringIfAbsent }, { publish });
|
||||
|
||||
await service.sendAlert({ ...payload, skill: undefined });
|
||||
|
||||
expect(setStringIfAbsent).toHaveBeenCalledWith(
|
||||
"lolympus-agent:alert:dedupe:latency",
|
||||
payload.timestamp,
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("notifies Slack without changing publish result", async () => {
|
||||
const setStringIfAbsent = vi.fn().mockResolvedValue(true);
|
||||
const publish = vi.fn().mockResolvedValue(undefined);
|
||||
const notify = vi.fn().mockResolvedValue(false);
|
||||
const service = new AlertService(
|
||||
{ setStringIfAbsent },
|
||||
{ publish },
|
||||
{ notify },
|
||||
);
|
||||
|
||||
const result = await service.sendAlert(payload);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(publish).toHaveBeenCalledWith(AGENT_ALERT_SUBJECT, payload);
|
||||
expect(notify).toHaveBeenCalledWith(payload);
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { AGENT_ALERT_SUBJECT, type AlertPayload } from "@common/nats-subjects";
|
||||
import type { NatsClient } from "@common/nats";
|
||||
import { getAlertRateLimitSec } from "../config.js";
|
||||
import { getLogger } from "./logger.js";
|
||||
import type { SlackNotifier } from "./slack-notifier.js";
|
||||
|
||||
export interface AlertSender {
|
||||
sendAlert(
|
||||
@@ -13,8 +14,11 @@ export interface AlertSender {
|
||||
): Promise<boolean>;
|
||||
}
|
||||
|
||||
function getAlertDedupeKey(kind: AlertPayload["kind"]): string {
|
||||
return `lolympus-agent:alert:dedupe:${kind}`;
|
||||
function getAlertDedupeKey(
|
||||
kind: AlertPayload["kind"],
|
||||
skill?: string,
|
||||
): string {
|
||||
return `lolympus-agent:alert:dedupe:${kind}${skill ? `:${skill}` : ""}`;
|
||||
}
|
||||
|
||||
export class AlertService implements AlertSender {
|
||||
@@ -23,6 +27,7 @@ export class AlertService implements AlertSender {
|
||||
constructor(
|
||||
private readonly garnetClient: Pick<GarnetClient, "setStringIfAbsent">,
|
||||
private readonly natsClient: Pick<NatsClient, "publish">,
|
||||
private readonly slackNotifier?: SlackNotifier | undefined,
|
||||
) {}
|
||||
|
||||
async sendAlert(
|
||||
@@ -31,7 +36,8 @@ export class AlertService implements AlertSender {
|
||||
dedupeKey?: string | undefined;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
const dedupeKey = options?.dedupeKey ?? getAlertDedupeKey(payload.kind);
|
||||
const dedupeKey =
|
||||
options?.dedupeKey ?? getAlertDedupeKey(payload.kind, payload.skill);
|
||||
const claimed = await this.garnetClient.setStringIfAbsent(
|
||||
dedupeKey,
|
||||
payload.timestamp,
|
||||
@@ -43,6 +49,7 @@ export class AlertService implements AlertSender {
|
||||
{
|
||||
dedupeKey,
|
||||
kind: payload.kind,
|
||||
skill: payload.skill,
|
||||
},
|
||||
"[Alert] Suppressed by rate limit",
|
||||
);
|
||||
@@ -55,12 +62,43 @@ export class AlertService implements AlertSender {
|
||||
kind: payload.kind,
|
||||
metric: payload.metric,
|
||||
severity: payload.severity,
|
||||
skill: payload.skill,
|
||||
threshold: payload.threshold,
|
||||
value: payload.value,
|
||||
window: payload.window,
|
||||
},
|
||||
"[Alert] Published",
|
||||
);
|
||||
this.notifySlack(payload);
|
||||
return true;
|
||||
}
|
||||
|
||||
private notifySlack(payload: AlertPayload): void {
|
||||
if (!this.slackNotifier) {
|
||||
return;
|
||||
}
|
||||
|
||||
void this.slackNotifier.notify(payload)
|
||||
.then((ok) => {
|
||||
if (!ok) {
|
||||
this.logger.warn(
|
||||
{
|
||||
kind: payload.kind,
|
||||
skill: payload.skill,
|
||||
},
|
||||
"[Alert] Slack notify failed",
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
this.logger.warn(
|
||||
{
|
||||
err: error,
|
||||
kind: payload.kind,
|
||||
skill: payload.skill,
|
||||
},
|
||||
"[Alert] Slack notify failed",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { handleMetricsRequest } from "./metrics-server.js";
|
||||
|
||||
describe("metrics server", () => {
|
||||
it("serves metrics and health checks", () => {
|
||||
const writeHead = vi.fn();
|
||||
const end = vi.fn();
|
||||
|
||||
handleMetricsRequest({ url: "/healthz" }, { end, writeHead });
|
||||
expect(writeHead).toHaveBeenCalledWith(200, { "Content-Type": "text/plain" });
|
||||
expect(end).toHaveBeenCalledWith("ok");
|
||||
|
||||
writeHead.mockClear();
|
||||
end.mockClear();
|
||||
|
||||
handleMetricsRequest({ url: "/metrics" }, { end, writeHead });
|
||||
expect(writeHead).toHaveBeenCalledWith(200, {
|
||||
"Content-Type": "text/plain; version=0.0.4",
|
||||
});
|
||||
expect(String(end.mock.calls[0]?.[0])).toContain(
|
||||
"# HELP agent_handoff_total",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,9 @@
|
||||
import { createServer, type Server } from "node:http";
|
||||
import {
|
||||
createServer,
|
||||
type IncomingMessage,
|
||||
type Server,
|
||||
type ServerResponse,
|
||||
} from "node:http";
|
||||
|
||||
import { getMetricsPort } from "../config.js";
|
||||
import { getLogger } from "./logger.js";
|
||||
@@ -6,23 +11,34 @@ import { renderMetrics } from "./metrics.js";
|
||||
|
||||
let server: Server | null = null;
|
||||
|
||||
export function handleMetricsRequest(
|
||||
request: Pick<IncomingMessage, "url">,
|
||||
response: Pick<ServerResponse, "end" | "writeHead">,
|
||||
): void {
|
||||
if (request.url === "/healthz") {
|
||||
response.writeHead(200, { "Content-Type": "text/plain" });
|
||||
response.end("ok");
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.url !== "/metrics") {
|
||||
response.writeHead(404);
|
||||
response.end("not found");
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "text/plain; version=0.0.4",
|
||||
});
|
||||
response.end(renderMetrics());
|
||||
}
|
||||
|
||||
export function startMetricsServer(port = getMetricsPort()): Server {
|
||||
if (server) {
|
||||
return server;
|
||||
}
|
||||
|
||||
server = createServer((request, response) => {
|
||||
if (request.url !== "/metrics") {
|
||||
response.writeHead(404);
|
||||
response.end("not found");
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "text/plain; version=0.0.4",
|
||||
});
|
||||
response.end(renderMetrics());
|
||||
});
|
||||
server = createServer(handleMetricsRequest);
|
||||
|
||||
server.listen(port, () => {
|
||||
getLogger().info({ port }, "✓ Metrics server listening");
|
||||
|
||||
@@ -26,15 +26,18 @@ describe("metrics", () => {
|
||||
metrics.handoffTotal.inc({
|
||||
from: "onboarding",
|
||||
reason: "out-of-scope",
|
||||
skill: "onboarding",
|
||||
to: "moderation",
|
||||
});
|
||||
metrics.handoffCount.observe(2);
|
||||
metrics.handoffCount.observe(2, { skill: "onboarding" });
|
||||
|
||||
const body = renderMetrics();
|
||||
|
||||
expect(body).toContain(
|
||||
'agent_handoff_total{from="onboarding",reason="out-of-scope",to="moderation"} 1',
|
||||
'agent_handoff_total{from="onboarding",reason="out-of-scope",skill="onboarding",to="moderation"} 1',
|
||||
);
|
||||
expect(body).toContain(
|
||||
'agent_handoff_count_bucket{skill="onboarding",le="2"} 1',
|
||||
);
|
||||
expect(body).toContain('agent_handoff_count_bucket{le="2"} 1');
|
||||
});
|
||||
});
|
||||
@@ -52,7 +52,7 @@ class CounterMetric {
|
||||
}
|
||||
|
||||
class HistogramMetric {
|
||||
private readonly values: number[] = [];
|
||||
private readonly samples = new Map<string, { labels: Labels; values: number[] }>();
|
||||
|
||||
constructor(
|
||||
readonly name: string,
|
||||
@@ -60,10 +60,15 @@ class HistogramMetric {
|
||||
private readonly buckets: number[],
|
||||
) {}
|
||||
|
||||
observe(value: number): void {
|
||||
if (Number.isFinite(value)) {
|
||||
this.values.push(value);
|
||||
observe(value: number, labels: Labels = {}): void {
|
||||
if (!Number.isFinite(value)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const key = labelKey(labels);
|
||||
const sample = this.samples.get(key) ?? { labels, values: [] };
|
||||
sample.values.push(value);
|
||||
this.samples.set(key, sample);
|
||||
}
|
||||
|
||||
render(): string[] {
|
||||
@@ -71,15 +76,31 @@ class HistogramMetric {
|
||||
`# HELP ${this.name} ${this.help}`,
|
||||
`# TYPE ${this.name} histogram`,
|
||||
];
|
||||
for (const bucket of this.buckets) {
|
||||
const count = this.values.filter((value) => value <= bucket).length;
|
||||
lines.push(`${this.name}_bucket{le="${bucket}"} ${count}`);
|
||||
|
||||
for (const sample of this.samples.values()) {
|
||||
for (const bucket of this.buckets) {
|
||||
const count = sample.values.filter((value) => value <= bucket).length;
|
||||
lines.push(
|
||||
`${this.name}_bucket${formatLabels({
|
||||
...sample.labels,
|
||||
le: bucket,
|
||||
})} ${count}`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
`${this.name}_bucket${formatLabels({
|
||||
...sample.labels,
|
||||
le: "+Inf",
|
||||
})} ${sample.values.length}`,
|
||||
);
|
||||
lines.push(
|
||||
`${this.name}_sum${formatLabels(sample.labels)} ${sample.values.reduce(
|
||||
(sum, value) => sum + value,
|
||||
0,
|
||||
)}`,
|
||||
);
|
||||
lines.push(`${this.name}_count${formatLabels(sample.labels)} ${sample.values.length}`);
|
||||
}
|
||||
lines.push(`${this.name}_bucket{le="+Inf"} ${this.values.length}`);
|
||||
lines.push(
|
||||
`${this.name}_sum ${this.values.reduce((sum, value) => sum + value, 0)}`,
|
||||
);
|
||||
lines.push(`${this.name}_count ${this.values.length}`);
|
||||
return lines;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import {
|
||||
formatSlackMessage,
|
||||
SlackWebhookNotifier,
|
||||
} from "./slack-notifier.js";
|
||||
|
||||
const payload: AlertPayload = {
|
||||
kind: "latency",
|
||||
metric: "shadow_eval.p95_latency_ms",
|
||||
severity: "critical",
|
||||
skill: "onboarding",
|
||||
suggestedAction: "Check endpoint latency.",
|
||||
threshold: 3000,
|
||||
timestamp: "2026-04-17T00:00:00.000Z",
|
||||
value: 4500,
|
||||
window: "5m",
|
||||
};
|
||||
|
||||
describe("formatSlackMessage", () => {
|
||||
it("formats severity, metric, skill, and action", () => {
|
||||
const message = formatSlackMessage(payload);
|
||||
|
||||
expect(message.text).toBe(
|
||||
":rotating_light: [latency] shadow_eval.p95_latency_ms",
|
||||
);
|
||||
expect(JSON.stringify(message.blocks)).toContain("*Skill*\\nonboarding");
|
||||
expect(JSON.stringify(message.blocks)).toContain("Check endpoint latency.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("SlackWebhookNotifier", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("posts the formatted Slack message", async () => {
|
||||
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(new Response("ok"));
|
||||
const notifier = new SlackWebhookNotifier(
|
||||
"https://hooks.slack.test/services/test",
|
||||
fetchImpl,
|
||||
);
|
||||
|
||||
await expect(notifier.notify(payload)).resolves.toBe(true);
|
||||
|
||||
const call = fetchImpl.mock.calls[0];
|
||||
expect(call?.[0]).toBe("https://hooks.slack.test/services/test");
|
||||
expect(call?.[1]?.body).toBe(JSON.stringify(formatSlackMessage(payload)));
|
||||
expect(call?.[1]?.headers).toEqual({ "Content-Type": "application/json" });
|
||||
expect(call?.[1]?.method).toBe("POST");
|
||||
expect(call?.[1]?.signal).toBeInstanceOf(AbortSignal);
|
||||
});
|
||||
|
||||
it("returns false for non-2xx responses", async () => {
|
||||
const fetchImpl = vi
|
||||
.fn<typeof fetch>()
|
||||
.mockResolvedValue(new Response("rate limited", { status: 429 }));
|
||||
const notifier = new SlackWebhookNotifier(
|
||||
"https://hooks.slack.test/services/test",
|
||||
fetchImpl,
|
||||
);
|
||||
|
||||
await expect(notifier.notify(payload)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when fetch rejects", async () => {
|
||||
const fetchImpl = vi.fn<typeof fetch>().mockRejectedValue(new Error("timeout"));
|
||||
const notifier = new SlackWebhookNotifier(
|
||||
"https://hooks.slack.test/services/test",
|
||||
fetchImpl,
|
||||
);
|
||||
|
||||
await expect(notifier.notify(payload)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("uses the configured timeout signal", async () => {
|
||||
vi.stubEnv("SLACK_WEBHOOK_TIMEOUT_MS", "1234");
|
||||
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(new Response("ok"));
|
||||
const notifier = new SlackWebhookNotifier(
|
||||
"https://hooks.slack.test/services/test",
|
||||
fetchImpl,
|
||||
);
|
||||
|
||||
await notifier.notify(payload);
|
||||
|
||||
expect(fetchImpl.mock.calls[0]?.[1]?.signal).toBeInstanceOf(AbortSignal);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { AlertPayload } from "@common/nats-subjects";
|
||||
import {
|
||||
getSlackAlertWebhookUrl,
|
||||
getSlackWebhookTimeoutMs,
|
||||
} from "../config.js";
|
||||
import { getLogger } from "./logger.js";
|
||||
|
||||
export interface SlackTextObject {
|
||||
text: string;
|
||||
type: "mrkdwn" | "plain_text";
|
||||
}
|
||||
|
||||
export interface SlackBlock {
|
||||
fields?: SlackTextObject[] | undefined;
|
||||
text?: SlackTextObject | undefined;
|
||||
type: "section";
|
||||
}
|
||||
|
||||
export interface SlackMessage {
|
||||
blocks: SlackBlock[];
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface SlackNotifier {
|
||||
notify(payload: AlertPayload): Promise<boolean>;
|
||||
}
|
||||
|
||||
const SEVERITY_EMOJI: Record<AlertPayload["severity"], string> = {
|
||||
critical: ":rotating_light:",
|
||||
info: ":information_source:",
|
||||
warn: ":warning:",
|
||||
};
|
||||
|
||||
function mrkdwn(text: string): SlackTextObject {
|
||||
return { text, type: "mrkdwn" };
|
||||
}
|
||||
|
||||
export function formatSlackMessage(payload: AlertPayload): SlackMessage {
|
||||
const blocks: SlackBlock[] = [
|
||||
{
|
||||
text: mrkdwn(
|
||||
`*${payload.severity.toUpperCase()}* - \`${payload.kind}\``,
|
||||
),
|
||||
type: "section",
|
||||
},
|
||||
{
|
||||
fields: [
|
||||
mrkdwn(`*Metric*\n${payload.metric}`),
|
||||
mrkdwn(`*Value / Threshold*\n${payload.value} / ${payload.threshold}`),
|
||||
mrkdwn(`*Window*\n${payload.window}`),
|
||||
mrkdwn(`*Skill*\n${payload.skill ?? "global"}`),
|
||||
],
|
||||
type: "section",
|
||||
},
|
||||
];
|
||||
|
||||
if (payload.suggestedAction?.trim()) {
|
||||
blocks.push({
|
||||
text: mrkdwn(`*Suggested action*\n${payload.suggestedAction}`),
|
||||
type: "section",
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
blocks,
|
||||
text: `${SEVERITY_EMOJI[payload.severity]} [${payload.kind}] ${payload.metric}`,
|
||||
};
|
||||
}
|
||||
|
||||
export class SlackWebhookNotifier implements SlackNotifier {
|
||||
constructor(
|
||||
private readonly webhookUrl: string,
|
||||
private readonly fetchImpl: typeof fetch = fetch,
|
||||
) {}
|
||||
|
||||
async notify(payload: AlertPayload): Promise<boolean> {
|
||||
try {
|
||||
const response = await this.fetchImpl(this.webhookUrl, {
|
||||
body: JSON.stringify(formatSlackMessage(payload)),
|
||||
headers: { "Content-Type": "application/json" },
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(getSlackWebhookTimeoutMs()),
|
||||
});
|
||||
return response.ok;
|
||||
} catch (error) {
|
||||
getLogger().warn(
|
||||
{
|
||||
err: error,
|
||||
kind: payload.kind,
|
||||
skill: payload.skill,
|
||||
},
|
||||
"[Slack] Notification failed",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createSlackNotifierFromEnv(): SlackNotifier | undefined {
|
||||
const webhookUrl = getSlackAlertWebhookUrl();
|
||||
return webhookUrl ? new SlackWebhookNotifier(webhookUrl) : undefined;
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import type { NatsClient } from "@common/nats";
|
||||
import type { LlmClient } from "@llm/llm-client";
|
||||
import { AlertService } from "@obs/alert";
|
||||
import { metrics } from "@obs/metrics";
|
||||
import { createSlackNotifierFromEnv } from "@obs/slack-notifier";
|
||||
import type { Skill } from "@skills/skill";
|
||||
import type { SkillCapabilityCard, SkillRegistry } from "@skills/skill-loader";
|
||||
import { getLogger } from "@obs/logger";
|
||||
@@ -52,7 +53,11 @@ export class SupervisorSkill implements Skill {
|
||||
private readonly llmClient?: Pick<LlmClient, "chatCompletion"> | undefined,
|
||||
private readonly skillRegistry?: SkillRegistry | undefined,
|
||||
) {
|
||||
this.alertService = new AlertService(garnetClient, natsClient);
|
||||
this.alertService = new AlertService(
|
||||
garnetClient,
|
||||
natsClient,
|
||||
createSlackNotifierFromEnv(),
|
||||
);
|
||||
}
|
||||
|
||||
canHandle(request: AgentRequest): boolean {
|
||||
@@ -83,13 +88,14 @@ export class SupervisorSkill implements Skill {
|
||||
target: target ?? "human-review",
|
||||
userId: request.userId,
|
||||
});
|
||||
metrics.humanOverrideTotal.inc({ kind: reason });
|
||||
metrics.humanOverrideTotal.inc({ kind: reason, skill: this.name });
|
||||
|
||||
await this.alertService.sendAlert(
|
||||
{
|
||||
kind: "supervisor-handoff",
|
||||
metric: "agent.supervisor_handoff",
|
||||
severity: "warn",
|
||||
skill: this.name,
|
||||
suggestedAction: "Review the supervisor handoff queue.",
|
||||
threshold: 0,
|
||||
timestamp: new Date().toISOString(),
|
||||
|
||||
@@ -21,6 +21,9 @@ const { getRawSecret } = await import("../common/secret-manager.js");
|
||||
const { ConversationStore } = await import("../conversation/index.js");
|
||||
const { ShadowWorker } = await import("../eval/shadow-worker.js");
|
||||
const { AlertService } = await import("../observability/alert.js");
|
||||
const { createSlackNotifierFromEnv } = await import(
|
||||
"../observability/slack-notifier.js"
|
||||
);
|
||||
const { initMongoDB, closeMongoDB } = await import("../common/mongodb.js");
|
||||
|
||||
function firstNonEmpty(
|
||||
@@ -175,7 +178,11 @@ async function bootstrap(): Promise<void> {
|
||||
providerApiKeys,
|
||||
});
|
||||
const conversationStore = new ConversationStore(garnetClient);
|
||||
const alertService = new AlertService(garnetClient, natsClient);
|
||||
const alertService = new AlertService(
|
||||
garnetClient,
|
||||
natsClient,
|
||||
createSlackNotifierFromEnv(),
|
||||
);
|
||||
const mongoConnectionString = firstNonEmpty(
|
||||
getRawSecret("MONGODB_CONNECTION_STRING"),
|
||||
process.env["MONGODB_CONNECTION_STRING"],
|
||||
|
||||
Reference in new issue
Block a user