Files
platform-state/platform/redis/overlays/main/monitoring.yaml
T

148 lines
6.5 KiB
YAML

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: redis
labels:
release: monitoring
spec:
selector:
matchLabels:
app.kubernetes.io/name: redis
app.kubernetes.io/component: metrics
endpoints:
- port: metrics
interval: 30s
scrapeTimeout: 10s
relabelings:
- targetLabel: job
replacement: redis
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: redis
labels:
release: monitoring
spec:
groups:
- name: redis
rules:
- alert: RedisUnavailable
expr: |
redis_up{namespace="platform",job="redis"} == 0
or up{namespace="platform",job="redis"} == 0
or absent(redis_up{namespace="platform",job="redis"})
for: 2m
labels:
severity: critical
annotations:
summary: Redis or its metrics endpoint is unavailable
description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target.
- alert: RedisMetricsScrapeFailed
expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0
for: 2m
labels:
severity: warning
annotations:
summary: Redis metrics collection is failing
description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics.
- alert: RedisAofWriteFailed
expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: Redis cannot persist writes to AOF
description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure.
- alert: RedisAofRewriteFailed
expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0
for: 5m
labels:
severity: warning
annotations:
summary: Redis AOF rewrite failed
description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds.
- alert: RedisWriteRejected
expr: |
increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0
or (
redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0
unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m
)
labels:
severity: critical
annotations:
summary: Redis rejected writes due to memory or persistence errors
description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly.
- alert: RedisMemoryNearMaxmemory
expr: |
(
redis_memory_used_bytes{namespace="platform",job="redis"}
- redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"}
) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85
for: 5m
labels:
severity: warning
annotations:
summary: Redis is approaching its maxmemory threshold
description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold.
- alert: RedisContainerMemoryHigh
expr: |
max by (namespace, pod, container) (
container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"}
)
/ max by (namespace, pod, container) (
kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"}
) > 0.85
for: 1m
labels:
severity: warning
annotations:
summary: Redis container is approaching its memory limit
description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting.
- alert: RedisContainerOomKilled
expr: |
max by (namespace, pod, container) (
kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1
)
and on (namespace, pod, container)
(
max by (namespace, pod, container) (
increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m])
) > 0
)
labels:
severity: critical
annotations:
summary: Redis was restarted after an OOM kill
description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks.
- alert: RedisAofFsyncDelayed
expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0
labels:
severity: warning
annotations:
summary: Redis AOF fsync is delayed
description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded.
- alert: RedisUnexpectedEviction
expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0
labels:
severity: critical
annotations:
summary: Redis evicted keys despite the intended noeviction policy
description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners.
- alert: RedisNodeDiskHeadroomLow
# local-path currently stores Redis on heize's root filesystem.
# If the storage path moves to a dedicated mount, change mountpoint.
expr: |
(
node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"}
* on (instance) group_left (nodename)
node_uname_info{job="node-exporter",nodename="heize"}
) < 10 * 1024 * 1024 * 1024
for: 5m
labels:
severity: warning
annotations:
summary: heize has less than 10GiB of free space for Redis AOF rewrites
description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space.