feat(redis): add shared Redis 8.10.3 to platform
This commit is contained in:
1 parent
eb96f56349
commit
67356b1489
13 files changed
+671
No files matched your search
@@ -0,0 +1,147 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: redis
|
||||
labels:
|
||||
release: monitoring
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis
|
||||
app.kubernetes.io/component: metrics
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
relabelings:
|
||||
- targetLabel: job
|
||||
replacement: redis
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: redis
|
||||
labels:
|
||||
release: monitoring
|
||||
spec:
|
||||
groups:
|
||||
- name: redis
|
||||
rules:
|
||||
- alert: RedisUnavailable
|
||||
expr: |
|
||||
redis_up{namespace="platform",job="redis"} == 0
|
||||
or up{namespace="platform",job="redis"} == 0
|
||||
or absent(redis_up{namespace="platform",job="redis"})
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis or its metrics endpoint is unavailable
|
||||
description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target.
|
||||
- alert: RedisMetricsScrapeFailed
|
||||
expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis metrics collection is failing
|
||||
description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics.
|
||||
- alert: RedisAofWriteFailed
|
||||
expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis cannot persist writes to AOF
|
||||
description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure.
|
||||
- alert: RedisAofRewriteFailed
|
||||
expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis AOF rewrite failed
|
||||
description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds.
|
||||
- alert: RedisWriteRejected
|
||||
expr: |
|
||||
increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0
|
||||
or (
|
||||
redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0
|
||||
unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m
|
||||
)
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis rejected writes due to memory or persistence errors
|
||||
description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly.
|
||||
- alert: RedisMemoryNearMaxmemory
|
||||
expr: |
|
||||
(
|
||||
redis_memory_used_bytes{namespace="platform",job="redis"}
|
||||
- redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"}
|
||||
) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis is approaching its maxmemory threshold
|
||||
description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold.
|
||||
- alert: RedisContainerMemoryHigh
|
||||
expr: |
|
||||
max by (namespace, pod, container) (
|
||||
container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"}
|
||||
)
|
||||
/ max by (namespace, pod, container) (
|
||||
kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"}
|
||||
) > 0.85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis container is approaching its memory limit
|
||||
description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting.
|
||||
- alert: RedisContainerOomKilled
|
||||
expr: |
|
||||
max by (namespace, pod, container) (
|
||||
kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1
|
||||
)
|
||||
and on (namespace, pod, container)
|
||||
(
|
||||
max by (namespace, pod, container) (
|
||||
increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m])
|
||||
) > 0
|
||||
)
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis was restarted after an OOM kill
|
||||
description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks.
|
||||
- alert: RedisAofFsyncDelayed
|
||||
expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis AOF fsync is delayed
|
||||
description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded.
|
||||
- alert: RedisUnexpectedEviction
|
||||
expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis evicted keys despite the intended noeviction policy
|
||||
description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners.
|
||||
- alert: RedisNodeDiskHeadroomLow
|
||||
# local-path currently stores Redis on heize's root filesystem.
|
||||
# If the storage path moves to a dedicated mount, change mountpoint.
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||
* on (instance) group_left (nodename)
|
||||
node_uname_info{job="node-exporter",nodename="heize"}
|
||||
) < 10 * 1024 * 1024 * 1024
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: heize has less than 10GiB of free space for Redis AOF rewrites
|
||||
description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space.
|
||||
Reference in new issue
Block a user