feat(redis): add shared Redis 8.10.3 to platform
This commit is contained in:
1 parent
eb96f56349
commit
67356b1489
13 files changed
+671
No files matched your search
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "5"
|
||||||
|
spec:
|
||||||
|
project: platform
|
||||||
|
source:
|
||||||
|
repoURL: https://git.dahyeon.us/dahyeon/platform-state.git
|
||||||
|
targetRevision: main
|
||||||
|
path: platform/redis/overlays/main
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: platform
|
||||||
@@ -97,3 +97,8 @@ apiVersion: v1
|
|||||||
kind: Namespace
|
kind: Namespace
|
||||||
metadata:
|
metadata:
|
||||||
name: make-space-prod
|
name: make-space-prod
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: platform
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# Redis
|
||||||
|
|
||||||
|
`platform` 네임스페이스에서 heize에 단일 인스턴스로 배포합니다.
|
||||||
|
|
||||||
|
- 내부 주소: `redis.platform.svc.cluster.local:6379` (ClusterIP)
|
||||||
|
- 저장소: `local-path` PVC `10Gi`
|
||||||
|
- 메모리: `maxmemory 3GiB`, 컨테이너 요청 `4Gi` / 제한 `5Gi`
|
||||||
|
- 영속화: AOF `everysec`
|
||||||
|
- 메모리 정책: `noeviction` — 한도 도달 시 메모리를 추가로 사용하는 쓰기 실패
|
||||||
|
|
||||||
|
## 배포
|
||||||
|
|
||||||
|
1. [Infisical 설정](overlays/main/infisical-secret.yaml)의 `prod` 환경,
|
||||||
|
`/platform/redis` 경로에 `REDIS_USERNAME`, `REDIS_PASSWORD`를 등록합니다.
|
||||||
|
사용자명은 영문·숫자·`_`·`.`·`-`만 허용하며 `default`는 사용할 수 없습니다.
|
||||||
|
2. main 반영 후 `cluster-main` 동기화로 `platform` 네임스페이스가 생성됐는지 확인합니다.
|
||||||
|
3. Argo CD의 [redis Application](../../argocd/applications/redis.yaml)을 수동 동기화합니다.
|
||||||
|
4. `platform/redis-auth` Secret 생성과 `redis-0` Pod의 Ready 상태를 확인합니다.
|
||||||
|
|
||||||
|
매니페스트 확인:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl kustomize platform/redis/overlays/main
|
||||||
|
```
|
||||||
|
|
||||||
|
## 개발자 접속
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n platform port-forward svc/redis 16379:6379
|
||||||
|
```
|
||||||
|
|
||||||
|
명령을 실행한 상태에서 `127.0.0.1:16379`에 Redis 사용자명·비밀번호로 접속합니다.
|
||||||
|
|
||||||
|
## 운영
|
||||||
|
|
||||||
|
- heize 장애 시 Redis도 중단됩니다. 초기 운영에서는 이를 감수합니다.
|
||||||
|
- `10Gi`는 요청량이며 실제 디스크 사용 상한이 아닙니다. PVC 삭제 시 데이터가 삭제될 수 있습니다.
|
||||||
|
- AOF는 별도 백업을 대신하지 않습니다. 재작성 중 메모리·디스크 사용량은 배포 후 측정합니다.
|
||||||
|
- [알림 규칙](overlays/main/monitoring.yaml)은 포함돼 있으며, 실제 수신 채널은 별도 설정해야 합니다.
|
||||||
|
- 현재 인증 정보 변경은 Infisical 동기화 후 Pod 재시작이 필요합니다.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n platform rollout restart statefulset/redis
|
||||||
|
kubectl -n platform rollout status statefulset/redis
|
||||||
|
```
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- statefulset.yaml
|
||||||
|
- service.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: redis-config
|
||||||
|
files:
|
||||||
|
- redis.conf
|
||||||
|
- start-redis.sh
|
||||||
|
|
||||||
|
labels:
|
||||||
|
- includeSelectors: true
|
||||||
|
pairs:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
app.kubernetes.io/part-of: redis
|
||||||
|
app.kubernetes.io/managed-by: kustomize
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
bind 0.0.0.0
|
||||||
|
protected-mode yes
|
||||||
|
port 6379
|
||||||
|
dir /data
|
||||||
|
aclfile /run/redis-acl/users.acl
|
||||||
|
|
||||||
|
appendonly yes
|
||||||
|
appendfsync everysec
|
||||||
|
save ""
|
||||||
|
|
||||||
|
# Redis "gb" uses binary units: 3gb = 3 GiB.
|
||||||
|
# The container limit is 5Gi. This is a measured-risk starting point: heavy
|
||||||
|
# writes during AOF rewrite can exhaust the remaining fork/COW headroom.
|
||||||
|
# maxmemory is not a hard limit on total process memory.
|
||||||
|
maxmemory 3gb
|
||||||
|
# Preserve keys until the application explicitly deletes or expires them.
|
||||||
|
maxmemory-policy noeviction
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis-headless
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: redis
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis-metrics
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: metrics
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
port: 9121
|
||||||
|
targetPort: metrics
|
||||||
|
# Keep the exporter scrapeable while the Redis readiness probe fails.
|
||||||
|
publishNotReadyAddresses: true
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: redis
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# Restrict usernames to a single ACL token; never interpolate raw passwords
|
||||||
|
# into Redis configuration syntax or command-line arguments.
|
||||||
|
case "${REDIS_USERNAME:-}" in
|
||||||
|
""|default|*[!a-zA-Z0-9_.-]*)
|
||||||
|
echo "REDIS_USERNAME must contain only letters, digits, _, . or - and must not be default." >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
if [ -z "${REDISCLI_AUTH:-}" ]; then
|
||||||
|
echo "REDIS_PASSWORD must not be empty." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
umask 077
|
||||||
|
password_hash="$(printf '%s' "$REDISCLI_AUTH" | sha256sum)"
|
||||||
|
password_hash="${password_hash%% *}"
|
||||||
|
printf 'user default off\nuser %s on #%s ~* &* +@all\n' \
|
||||||
|
"$REDIS_USERNAME" "$password_hash" > /run/redis-acl/users.acl
|
||||||
|
unset password_hash
|
||||||
|
|
||||||
|
exec redis-server /etc/redis/redis.conf
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
spec:
|
||||||
|
serviceName: redis-headless
|
||||||
|
replicas: 1
|
||||||
|
persistentVolumeClaimRetentionPolicy:
|
||||||
|
whenDeleted: Retain
|
||||||
|
whenScaled: Retain
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 999
|
||||||
|
runAsGroup: 1000
|
||||||
|
fsGroup: 1000
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: redis
|
||||||
|
image: redis:8.10.3-alpine
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- /etc/redis/start-redis.sh
|
||||||
|
env:
|
||||||
|
- name: REDIS_USERNAME
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: username
|
||||||
|
- name: REDISCLI_AUTH
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: password
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
containerPort: 6379
|
||||||
|
startupProbe:
|
||||||
|
exec:
|
||||||
|
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 120
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 4Gi
|
||||||
|
limits:
|
||||||
|
memory: 5Gi
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
capabilities:
|
||||||
|
drop: [ALL]
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/redis
|
||||||
|
readOnly: true
|
||||||
|
- name: acl
|
||||||
|
mountPath: /run/redis-acl
|
||||||
|
- name: exporter
|
||||||
|
image: oliver006/redis_exporter:v1.94.0
|
||||||
|
args:
|
||||||
|
- --redis.addr=redis://127.0.0.1:6379
|
||||||
|
- --redis-only-metrics
|
||||||
|
- --disable-scrape-endpoint
|
||||||
|
- --connection-timeout=5s
|
||||||
|
env:
|
||||||
|
- name: REDIS_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: username
|
||||||
|
- name: REDIS_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: password
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
containerPort: 9121
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 25m
|
||||||
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop: [ALL]
|
||||||
|
volumes:
|
||||||
|
- name: acl
|
||||||
|
emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 1Mi
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: redis-config
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: data
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: local-path
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
apiVersion: secrets.infisical.com/v1beta1
|
||||||
|
kind: InfisicalStaticSecret
|
||||||
|
metadata:
|
||||||
|
name: redis-auth
|
||||||
|
spec:
|
||||||
|
infisicalAuthRef:
|
||||||
|
name: infisical-auth
|
||||||
|
namespace: infisical
|
||||||
|
syncOptions:
|
||||||
|
refreshInterval: 60s
|
||||||
|
instantUpdates: false
|
||||||
|
sources:
|
||||||
|
- projectId: 8520b8c0-3796-4fd6-a03c-6f0508c8c42c
|
||||||
|
environmentSlug: prod
|
||||||
|
secretPath: /platform/redis
|
||||||
|
recursive: false
|
||||||
|
targets:
|
||||||
|
- name: redis-auth
|
||||||
|
namespace: platform
|
||||||
|
kind: Secret
|
||||||
|
creationPolicy: Owner
|
||||||
|
secretType: Opaque
|
||||||
|
template:
|
||||||
|
engineVersion: v1
|
||||||
|
data:
|
||||||
|
username: "{{ .REDIS_USERNAME.Value }}"
|
||||||
|
password: "{{ .REDIS_PASSWORD.Value }}"
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
namespace: platform
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- ../../base
|
||||||
|
- infisical-secret.yaml
|
||||||
|
- monitoring.yaml
|
||||||
|
|
||||||
|
patches:
|
||||||
|
- path: statefulset-patch.yaml
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
labels:
|
||||||
|
release: monitoring
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
app.kubernetes.io/component: metrics
|
||||||
|
endpoints:
|
||||||
|
- port: metrics
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
|
relabelings:
|
||||||
|
- targetLabel: job
|
||||||
|
replacement: redis
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
labels:
|
||||||
|
release: monitoring
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: redis
|
||||||
|
rules:
|
||||||
|
- alert: RedisUnavailable
|
||||||
|
expr: |
|
||||||
|
redis_up{namespace="platform",job="redis"} == 0
|
||||||
|
or up{namespace="platform",job="redis"} == 0
|
||||||
|
or absent(redis_up{namespace="platform",job="redis"})
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis or its metrics endpoint is unavailable
|
||||||
|
description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target.
|
||||||
|
- alert: RedisMetricsScrapeFailed
|
||||||
|
expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis metrics collection is failing
|
||||||
|
description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics.
|
||||||
|
- alert: RedisAofWriteFailed
|
||||||
|
expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis cannot persist writes to AOF
|
||||||
|
description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure.
|
||||||
|
- alert: RedisAofRewriteFailed
|
||||||
|
expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis AOF rewrite failed
|
||||||
|
description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds.
|
||||||
|
- alert: RedisWriteRejected
|
||||||
|
expr: |
|
||||||
|
increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0
|
||||||
|
or (
|
||||||
|
redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0
|
||||||
|
unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m
|
||||||
|
)
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis rejected writes due to memory or persistence errors
|
||||||
|
description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly.
|
||||||
|
- alert: RedisMemoryNearMaxmemory
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
redis_memory_used_bytes{namespace="platform",job="redis"}
|
||||||
|
- redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"}
|
||||||
|
) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis is approaching its maxmemory threshold
|
||||||
|
description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold.
|
||||||
|
- alert: RedisContainerMemoryHigh
|
||||||
|
expr: |
|
||||||
|
max by (namespace, pod, container) (
|
||||||
|
container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"}
|
||||||
|
)
|
||||||
|
/ max by (namespace, pod, container) (
|
||||||
|
kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"}
|
||||||
|
) > 0.85
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis container is approaching its memory limit
|
||||||
|
description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting.
|
||||||
|
- alert: RedisContainerOomKilled
|
||||||
|
expr: |
|
||||||
|
max by (namespace, pod, container) (
|
||||||
|
kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1
|
||||||
|
)
|
||||||
|
and on (namespace, pod, container)
|
||||||
|
(
|
||||||
|
max by (namespace, pod, container) (
|
||||||
|
increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m])
|
||||||
|
) > 0
|
||||||
|
)
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis was restarted after an OOM kill
|
||||||
|
description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks.
|
||||||
|
- alert: RedisAofFsyncDelayed
|
||||||
|
expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis AOF fsync is delayed
|
||||||
|
description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded.
|
||||||
|
- alert: RedisUnexpectedEviction
|
||||||
|
expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis evicted keys despite the intended noeviction policy
|
||||||
|
description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners.
|
||||||
|
- alert: RedisNodeDiskHeadroomLow
|
||||||
|
# local-path currently stores Redis on heize's root filesystem.
|
||||||
|
# If the storage path moves to a dedicated mount, change mountpoint.
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||||
|
* on (instance) group_left (nodename)
|
||||||
|
node_uname_info{job="node-exporter",nodename="heize"}
|
||||||
|
) < 10 * 1024 * 1024 * 1024
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: heize has less than 10GiB of free space for Redis AOF rewrites
|
||||||
|
description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/environment: prod
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: heize
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
"""Validate rendered Redis resources and exercise alert behavior with promtool.
|
||||||
|
|
||||||
|
Requires kubectl and promtool on PATH (or --promtool /path/to/promtool).
|
||||||
|
Uses client dry-run only; never applies resources to the cluster.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
|
||||||
|
def parse_json_stream(text):
|
||||||
|
decoder = json.JSONDecoder()
|
||||||
|
documents = []
|
||||||
|
while text.strip():
|
||||||
|
text = text.lstrip()
|
||||||
|
document, end = decoder.raw_decode(text)
|
||||||
|
documents.append(document)
|
||||||
|
text = text[end:]
|
||||||
|
return documents
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--promtool", default="promtool")
|
||||||
|
args = parser.parse_args()
|
||||||
|
app = Path(__file__).resolve().parents[1]
|
||||||
|
result = subprocess.run(
|
||||||
|
["kubectl", "create", "--dry-run=client", "-k",
|
||||||
|
str(app / "overlays/main"), "-o", "json", "--request-timeout=10s"],
|
||||||
|
check=True, capture_output=True, text=True, encoding="utf-8",
|
||||||
|
)
|
||||||
|
resources = parse_json_stream(result.stdout)
|
||||||
|
sts = next(r for r in resources if r["kind"] == "StatefulSet")
|
||||||
|
containers = sts["spec"]["template"]["spec"]["containers"]
|
||||||
|
redis = next(c for c in containers if c["name"] == "redis")
|
||||||
|
assert redis["resources"]["limits"]["memory"] == "5Gi"
|
||||||
|
assert redis["resources"]["requests"]["memory"] == "4Gi"
|
||||||
|
assert all("cpu" not in c["resources"].get("limits", {}) for c in containers)
|
||||||
|
config = next(r for r in resources if r["kind"] == "ConfigMap")
|
||||||
|
assert "maxmemory 3gb\n" in config["data"]["redis.conf"]
|
||||||
|
assert "maxmemory-policy noeviction\n" in config["data"]["redis.conf"]
|
||||||
|
metrics_service = next(
|
||||||
|
r for r in resources if r["kind"] == "Service"
|
||||||
|
and r["metadata"]["name"] == "redis-metrics"
|
||||||
|
)
|
||||||
|
assert metrics_service["spec"]["publishNotReadyAddresses"] is True
|
||||||
|
monitor = next(r for r in resources if r["kind"] == "ServiceMonitor")
|
||||||
|
rule_resource = next(r for r in resources if r["kind"] == "PrometheusRule")
|
||||||
|
assert monitor["metadata"]["labels"]["release"] == "monitoring"
|
||||||
|
assert rule_resource["metadata"]["labels"]["release"] == "monitoring"
|
||||||
|
assert any(
|
||||||
|
relabel.get("targetLabel") == "job" and relabel.get("replacement") == "redis"
|
||||||
|
for relabel in monitor["spec"]["endpoints"][0]["relabelings"]
|
||||||
|
)
|
||||||
|
rules = {
|
||||||
|
rule["alert"]: rule
|
||||||
|
for group in rule_resource["spec"]["groups"]
|
||||||
|
for rule in group["rules"]
|
||||||
|
}
|
||||||
|
base = {"namespace": "platform", "job": "redis"}
|
||||||
|
container = {"namespace": "platform", "pod": "redis-0", "container": "redis"}
|
||||||
|
|
||||||
|
def series(metric, values, labels=None):
|
||||||
|
labels = base if labels is None else labels
|
||||||
|
selector = ",".join(f'{key}="{value}"' for key, value in labels.items())
|
||||||
|
return {"series": f"{metric}{{{selector}}}", "values": values}
|
||||||
|
|
||||||
|
tests = []
|
||||||
|
|
||||||
|
def case(name, alert, inputs, firing, labels=None, at="6m", healthy=True):
|
||||||
|
labels = base if labels is None else labels
|
||||||
|
expected = []
|
||||||
|
if firing:
|
||||||
|
expected = [{
|
||||||
|
"exp_labels": {**labels, **rules[alert]["labels"]},
|
||||||
|
"exp_annotations": rules[alert]["annotations"],
|
||||||
|
}]
|
||||||
|
tests.append({
|
||||||
|
"name": name, "interval": "30s",
|
||||||
|
"input_series": (
|
||||||
|
[series("redis_up", "1+0x30")] if healthy else []
|
||||||
|
) + inputs,
|
||||||
|
"alert_rule_test": [{
|
||||||
|
"eval_time": at, "alertname": alert, "exp_alerts": expected,
|
||||||
|
}],
|
||||||
|
})
|
||||||
|
|
||||||
|
case("healthy Redis", "RedisUnavailable", [], False)
|
||||||
|
case("missing exporter target", "RedisUnavailable", [], True, healthy=False)
|
||||||
|
case("Redis unreachable", "RedisUnavailable",
|
||||||
|
[series("redis_up", "0+0x30")], True, healthy=False)
|
||||||
|
case("HTTP endpoint down while Redis metric still exists", "RedisUnavailable",
|
||||||
|
[series("up", "0+0x30")], True)
|
||||||
|
case("brief outage is below hold duration", "RedisUnavailable",
|
||||||
|
[series("redis_up", "0+0x30")], False, at="1m", healthy=False)
|
||||||
|
case("exporter cannot read Redis INFO", "RedisMetricsScrapeFailed",
|
||||||
|
[series("redis_exporter_last_scrape_error", "1+0x30", {**base, "err": "NOPERM"})],
|
||||||
|
True, {**base, "err": "NOPERM"})
|
||||||
|
for metric, alert in [
|
||||||
|
("redis_aof_last_write_status", "RedisAofWriteFailed"),
|
||||||
|
("redis_aof_last_bgrewrite_status", "RedisAofRewriteFailed"),
|
||||||
|
]:
|
||||||
|
case(f"{metric} ok is one", alert, [series(metric, "1+0x30")], False)
|
||||||
|
case(f"{metric} failure is zero", alert, [series(metric, "0+0x30")], True)
|
||||||
|
for error in ["OOM", "MISCONF"]:
|
||||||
|
labels = {**base, "err": error}
|
||||||
|
case(f"new {error} counter starts at one", "RedisWriteRejected",
|
||||||
|
[series("redis_errors_total", "_ _ _ _ 1+0x26", labels)],
|
||||||
|
True, labels, at="3m")
|
||||||
|
case(f"old {error} error does not alert forever", "RedisWriteRejected",
|
||||||
|
[series("redis_errors_total", "1+0x30", labels)],
|
||||||
|
False, at="10m")
|
||||||
|
case("growing OOM error counter", "RedisWriteRejected",
|
||||||
|
[series("redis_errors_total", "0+1x30", {**base, "err": "OOM"})],
|
||||||
|
True, {**base, "err": "OOM"})
|
||||||
|
case("Redis memory high", "RedisMemoryNearMaxmemory", [
|
||||||
|
series("redis_memory_used_bytes", "3006477107+0x30"),
|
||||||
|
series("redis_mem_not_counted_for_eviction_bytes", "0+0x30"),
|
||||||
|
series("redis_memory_max_bytes", "3221225472+0x30"),
|
||||||
|
], True)
|
||||||
|
case("exclude buffers not counted by eviction", "RedisMemoryNearMaxmemory", [
|
||||||
|
series("redis_memory_used_bytes", "3221225472+0x30"),
|
||||||
|
series("redis_mem_not_counted_for_eviction_bytes", "1073741824+0x30"),
|
||||||
|
series("redis_memory_max_bytes", "3221225472+0x30"),
|
||||||
|
], False)
|
||||||
|
case("container limit evaluated independently from maxmemory", "RedisContainerMemoryHigh", [
|
||||||
|
series("container_memory_working_set_bytes", "4831838208+0x30", container),
|
||||||
|
series("kube_pod_container_resource_limits", "5368709120+0x30",
|
||||||
|
{**container, "resource": "memory"}),
|
||||||
|
], True, container)
|
||||||
|
for restarts, firing in [("0+1x30", True), ("1+0x30", False)]:
|
||||||
|
case(f"OOM reason with restarts {restarts}", "RedisContainerOomKilled", [
|
||||||
|
series("kube_pod_container_status_last_terminated_reason", "1+0x30",
|
||||||
|
{**container, "reason": "OOMKilled"}),
|
||||||
|
series("kube_pod_container_status_restarts_total", restarts, container),
|
||||||
|
], firing, container, at="12m")
|
||||||
|
for metric, alert in [
|
||||||
|
("redis_aof_delayed_fsync", "RedisAofFsyncDelayed"),
|
||||||
|
("redis_evicted_keys_total", "RedisUnexpectedEviction"),
|
||||||
|
]:
|
||||||
|
case(f"increasing {metric}", alert, [series(metric, "0+1x30")], True)
|
||||||
|
case(f"stable {metric}", alert, [series(metric, "1+0x30")], False)
|
||||||
|
disk = {"job": "node-exporter", "instance": "10.250.0.16:9100",
|
||||||
|
"mountpoint": "/", "fstype": "ext4", "device": "/dev/sda1"}
|
||||||
|
for node, free, firing in [
|
||||||
|
("heize", "9663676416", True),
|
||||||
|
("heize", "12884901888", False),
|
||||||
|
("suga", "9663676416", False),
|
||||||
|
]:
|
||||||
|
case(f"{node} free disk {free}", "RedisNodeDiskHeadroomLow", [
|
||||||
|
series("node_filesystem_avail_bytes", free + "+0x30", disk),
|
||||||
|
series("node_uname_info", "1+0x30",
|
||||||
|
{"job": "node-exporter", "instance": disk["instance"], "nodename": node}),
|
||||||
|
], firing, {**disk, "nodename": node})
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix="redis-monitoring-") as directory:
|
||||||
|
rule_path = Path(directory) / "rules.json"
|
||||||
|
rule_path.write_text(json.dumps(rule_resource["spec"]), encoding="utf-8")
|
||||||
|
test_path = Path(directory) / "tests.json"
|
||||||
|
test_path.write_text(json.dumps({
|
||||||
|
"rule_files": [rule_path.as_posix()],
|
||||||
|
"evaluation_interval": "30s", "tests": tests,
|
||||||
|
}), encoding="utf-8")
|
||||||
|
subprocess.run([args.promtool, "check", "rules", str(rule_path)], check=True)
|
||||||
|
subprocess.run([args.promtool, "test", "rules", str(test_path)], check=True)
|
||||||
|
print(f"PASS: resource assertions and {len(tests)} alert scenarios")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in new issue
Block a user