148 lines
6.5 KiB
YAML
148 lines
6.5 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: ServiceMonitor
|
|
metadata:
|
|
name: redis
|
|
labels:
|
|
release: monitoring
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis
|
|
app.kubernetes.io/component: metrics
|
|
endpoints:
|
|
- port: metrics
|
|
interval: 30s
|
|
scrapeTimeout: 10s
|
|
relabelings:
|
|
- targetLabel: job
|
|
replacement: redis
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: redis
|
|
labels:
|
|
release: monitoring
|
|
spec:
|
|
groups:
|
|
- name: redis
|
|
rules:
|
|
- alert: RedisUnavailable
|
|
expr: |
|
|
redis_up{namespace="platform",job="redis"} == 0
|
|
or up{namespace="platform",job="redis"} == 0
|
|
or absent(redis_up{namespace="platform",job="redis"})
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Redis or its metrics endpoint is unavailable
|
|
description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target.
|
|
- alert: RedisMetricsScrapeFailed
|
|
expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Redis metrics collection is failing
|
|
description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics.
|
|
- alert: RedisAofWriteFailed
|
|
expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Redis cannot persist writes to AOF
|
|
description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure.
|
|
- alert: RedisAofRewriteFailed
|
|
expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Redis AOF rewrite failed
|
|
description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds.
|
|
- alert: RedisWriteRejected
|
|
expr: |
|
|
increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0
|
|
or (
|
|
redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0
|
|
unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m
|
|
)
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Redis rejected writes due to memory or persistence errors
|
|
description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly.
|
|
- alert: RedisMemoryNearMaxmemory
|
|
expr: |
|
|
(
|
|
redis_memory_used_bytes{namespace="platform",job="redis"}
|
|
- redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"}
|
|
) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Redis is approaching its maxmemory threshold
|
|
description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold.
|
|
- alert: RedisContainerMemoryHigh
|
|
expr: |
|
|
max by (namespace, pod, container) (
|
|
container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"}
|
|
)
|
|
/ max by (namespace, pod, container) (
|
|
kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"}
|
|
) > 0.85
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Redis container is approaching its memory limit
|
|
description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting.
|
|
- alert: RedisContainerOomKilled
|
|
expr: |
|
|
max by (namespace, pod, container) (
|
|
kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1
|
|
)
|
|
and on (namespace, pod, container)
|
|
(
|
|
max by (namespace, pod, container) (
|
|
increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m])
|
|
) > 0
|
|
)
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Redis was restarted after an OOM kill
|
|
description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks.
|
|
- alert: RedisAofFsyncDelayed
|
|
expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Redis AOF fsync is delayed
|
|
description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded.
|
|
- alert: RedisUnexpectedEviction
|
|
expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Redis evicted keys despite the intended noeviction policy
|
|
description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners.
|
|
- alert: RedisNodeDiskHeadroomLow
|
|
# local-path currently stores Redis on heize's root filesystem.
|
|
# If the storage path moves to a dedicated mount, change mountpoint.
|
|
expr: |
|
|
(
|
|
node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"}
|
|
* on (instance) group_left (nodename)
|
|
node_uname_info{job="node-exporter",nodename="heize"}
|
|
) < 10 * 1024 * 1024 * 1024
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: heize has less than 10GiB of free space for Redis AOF rewrites
|
|
description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space.
|