feat(redis): add shared Redis 8.10.3 to platform
This commit is contained in:
1 parent
eb96f56349
commit
67356b1489
13 files changed
+671
No files matched your search
@@ -0,0 +1,27 @@
|
||||
apiVersion: secrets.infisical.com/v1beta1
|
||||
kind: InfisicalStaticSecret
|
||||
metadata:
|
||||
name: redis-auth
|
||||
spec:
|
||||
infisicalAuthRef:
|
||||
name: infisical-auth
|
||||
namespace: infisical
|
||||
syncOptions:
|
||||
refreshInterval: 60s
|
||||
instantUpdates: false
|
||||
sources:
|
||||
- projectId: 8520b8c0-3796-4fd6-a03c-6f0508c8c42c
|
||||
environmentSlug: prod
|
||||
secretPath: /platform/redis
|
||||
recursive: false
|
||||
targets:
|
||||
- name: redis-auth
|
||||
namespace: platform
|
||||
kind: Secret
|
||||
creationPolicy: Owner
|
||||
secretType: Opaque
|
||||
template:
|
||||
engineVersion: v1
|
||||
data:
|
||||
username: "{{ .REDIS_USERNAME.Value }}"
|
||||
password: "{{ .REDIS_PASSWORD.Value }}"
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: platform
|
||||
|
||||
resources:
|
||||
- ../../base
|
||||
- infisical-secret.yaml
|
||||
- monitoring.yaml
|
||||
|
||||
patches:
|
||||
- path: statefulset-patch.yaml
|
||||
@@ -0,0 +1,147 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: redis
|
||||
labels:
|
||||
release: monitoring
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis
|
||||
app.kubernetes.io/component: metrics
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
relabelings:
|
||||
- targetLabel: job
|
||||
replacement: redis
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: redis
|
||||
labels:
|
||||
release: monitoring
|
||||
spec:
|
||||
groups:
|
||||
- name: redis
|
||||
rules:
|
||||
- alert: RedisUnavailable
|
||||
expr: |
|
||||
redis_up{namespace="platform",job="redis"} == 0
|
||||
or up{namespace="platform",job="redis"} == 0
|
||||
or absent(redis_up{namespace="platform",job="redis"})
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis or its metrics endpoint is unavailable
|
||||
description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target.
|
||||
- alert: RedisMetricsScrapeFailed
|
||||
expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis metrics collection is failing
|
||||
description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics.
|
||||
- alert: RedisAofWriteFailed
|
||||
expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis cannot persist writes to AOF
|
||||
description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure.
|
||||
- alert: RedisAofRewriteFailed
|
||||
expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis AOF rewrite failed
|
||||
description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds.
|
||||
- alert: RedisWriteRejected
|
||||
expr: |
|
||||
increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0
|
||||
or (
|
||||
redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0
|
||||
unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m
|
||||
)
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis rejected writes due to memory or persistence errors
|
||||
description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly.
|
||||
- alert: RedisMemoryNearMaxmemory
|
||||
expr: |
|
||||
(
|
||||
redis_memory_used_bytes{namespace="platform",job="redis"}
|
||||
- redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"}
|
||||
) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis is approaching its maxmemory threshold
|
||||
description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold.
|
||||
- alert: RedisContainerMemoryHigh
|
||||
expr: |
|
||||
max by (namespace, pod, container) (
|
||||
container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"}
|
||||
)
|
||||
/ max by (namespace, pod, container) (
|
||||
kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"}
|
||||
) > 0.85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis container is approaching its memory limit
|
||||
description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting.
|
||||
- alert: RedisContainerOomKilled
|
||||
expr: |
|
||||
max by (namespace, pod, container) (
|
||||
kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1
|
||||
)
|
||||
and on (namespace, pod, container)
|
||||
(
|
||||
max by (namespace, pod, container) (
|
||||
increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m])
|
||||
) > 0
|
||||
)
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis was restarted after an OOM kill
|
||||
description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks.
|
||||
- alert: RedisAofFsyncDelayed
|
||||
expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Redis AOF fsync is delayed
|
||||
description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded.
|
||||
- alert: RedisUnexpectedEviction
|
||||
expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Redis evicted keys despite the intended noeviction policy
|
||||
description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners.
|
||||
- alert: RedisNodeDiskHeadroomLow
|
||||
# local-path currently stores Redis on heize's root filesystem.
|
||||
# If the storage path moves to a dedicated mount, change mountpoint.
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||
* on (instance) group_left (nodename)
|
||||
node_uname_info{job="node-exporter",nodename="heize"}
|
||||
) < 10 * 1024 * 1024 * 1024
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: heize has less than 10GiB of free space for Redis AOF rewrites
|
||||
description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space.
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: redis
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/environment: prod
|
||||
spec:
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: heize
|
||||
Reference in new issue
Block a user