diff --git a/argocd/applications/redis.yaml b/argocd/applications/redis.yaml new file mode 100644 index 0000000..3329cc5 --- /dev/null +++ b/argocd/applications/redis.yaml @@ -0,0 +1,16 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: redis + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + project: platform + source: + repoURL: https://git.dahyeon.us/dahyeon/platform-state.git + targetRevision: main + path: platform/redis/overlays/main + destination: + server: https://kubernetes.default.svc + namespace: platform diff --git a/clusters/main/namespaces.yaml b/clusters/main/namespaces.yaml index 81fef4d..5596668 100644 --- a/clusters/main/namespaces.yaml +++ b/clusters/main/namespaces.yaml @@ -97,3 +97,8 @@ apiVersion: v1 kind: Namespace metadata: name: make-space-prod +--- +apiVersion: v1 +kind: Namespace +metadata: + name: platform diff --git a/platform/redis/README.md b/platform/redis/README.md new file mode 100644 index 0000000..a94fa19 --- /dev/null +++ b/platform/redis/README.md @@ -0,0 +1,45 @@ +# Redis + +`platform` 네임스페이스에서 heize에 단일 인스턴스로 배포합니다. + +- 내부 주소: `redis.platform.svc.cluster.local:6379` (ClusterIP) +- 저장소: `local-path` PVC `10Gi` +- 메모리: `maxmemory 3GiB`, 컨테이너 요청 `4Gi` / 제한 `5Gi` +- 영속화: AOF `everysec` +- 메모리 정책: `noeviction` — 한도 도달 시 메모리를 추가로 사용하는 쓰기 실패 + +## 배포 + +1. [Infisical 설정](overlays/main/infisical-secret.yaml)의 `prod` 환경, + `/platform/redis` 경로에 `REDIS_USERNAME`, `REDIS_PASSWORD`를 등록합니다. + 사용자명은 영문·숫자·`_`·`.`·`-`만 허용하며 `default`는 사용할 수 없습니다. +2. main 반영 후 `cluster-main` 동기화로 `platform` 네임스페이스가 생성됐는지 확인합니다. +3. Argo CD의 [redis Application](../../argocd/applications/redis.yaml)을 수동 동기화합니다. +4. `platform/redis-auth` Secret 생성과 `redis-0` Pod의 Ready 상태를 확인합니다. + +매니페스트 확인: + +```sh +kubectl kustomize platform/redis/overlays/main +``` + +## 개발자 접속 + +```sh +kubectl -n platform port-forward svc/redis 16379:6379 +``` + +명령을 실행한 상태에서 `127.0.0.1:16379`에 Redis 사용자명·비밀번호로 접속합니다. + +## 운영 + +- heize 장애 시 Redis도 중단됩니다. 초기 운영에서는 이를 감수합니다. +- `10Gi`는 요청량이며 실제 디스크 사용 상한이 아닙니다. PVC 삭제 시 데이터가 삭제될 수 있습니다. +- AOF는 별도 백업을 대신하지 않습니다. 재작성 중 메모리·디스크 사용량은 배포 후 측정합니다. +- [알림 규칙](overlays/main/monitoring.yaml)은 포함돼 있으며, 실제 수신 채널은 별도 설정해야 합니다. +- 현재 인증 정보 변경은 Infisical 동기화 후 Pod 재시작이 필요합니다. + +```sh +kubectl -n platform rollout restart statefulset/redis +kubectl -n platform rollout status statefulset/redis +``` diff --git a/platform/redis/base/kustomization.yaml b/platform/redis/base/kustomization.yaml new file mode 100644 index 0000000..8ea24c7 --- /dev/null +++ b/platform/redis/base/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - statefulset.yaml + - service.yaml + +configMapGenerator: + - name: redis-config + files: + - redis.conf + - start-redis.sh + +labels: + - includeSelectors: true + pairs: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: redis + app.kubernetes.io/managed-by: kustomize diff --git a/platform/redis/base/redis.conf b/platform/redis/base/redis.conf new file mode 100644 index 0000000..0d21e36 --- /dev/null +++ b/platform/redis/base/redis.conf @@ -0,0 +1,17 @@ +bind 0.0.0.0 +protected-mode yes +port 6379 +dir /data +aclfile /run/redis-acl/users.acl + +appendonly yes +appendfsync everysec +save "" + +# Redis "gb" uses binary units: 3gb = 3 GiB. +# The container limit is 5Gi. This is a measured-risk starting point: heavy +# writes during AOF rewrite can exhaust the remaining fork/COW headroom. +# maxmemory is not a hard limit on total process memory. +maxmemory 3gb +# Preserve keys until the application explicitly deletes or expires them. +maxmemory-policy noeviction diff --git a/platform/redis/base/service.yaml b/platform/redis/base/service.yaml new file mode 100644 index 0000000..552c49b --- /dev/null +++ b/platform/redis/base/service.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-headless +spec: + clusterIP: None + selector: + app.kubernetes.io/name: redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-metrics + labels: + app.kubernetes.io/component: metrics +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: redis + ports: + - name: metrics + port: 9121 + targetPort: metrics + # Keep the exporter scrapeable while the Redis readiness probe fails. + publishNotReadyAddresses: true +--- +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: redis + ports: + - name: redis + port: 6379 + targetPort: redis diff --git a/platform/redis/base/start-redis.sh b/platform/redis/base/start-redis.sh new file mode 100644 index 0000000..0d7886f --- /dev/null +++ b/platform/redis/base/start-redis.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu + +# Restrict usernames to a single ACL token; never interpolate raw passwords +# into Redis configuration syntax or command-line arguments. +case "${REDIS_USERNAME:-}" in + ""|default|*[!a-zA-Z0-9_.-]*) + echo "REDIS_USERNAME must contain only letters, digits, _, . or - and must not be default." >&2 + exit 1 + ;; +esac +if [ -z "${REDISCLI_AUTH:-}" ]; then + echo "REDIS_PASSWORD must not be empty." >&2 + exit 1 +fi + +umask 077 +password_hash="$(printf '%s' "$REDISCLI_AUTH" | sha256sum)" +password_hash="${password_hash%% *}" +printf 'user default off\nuser %s on #%s ~* &* +@all\n' \ + "$REDIS_USERNAME" "$password_hash" > /run/redis-acl/users.acl +unset password_hash + +exec redis-server /etc/redis/redis.conf diff --git a/platform/redis/base/statefulset.yaml b/platform/redis/base/statefulset.yaml new file mode 100644 index 0000000..fd12666 --- /dev/null +++ b/platform/redis/base/statefulset.yaml @@ -0,0 +1,132 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis +spec: + serviceName: redis-headless + replicas: 1 + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + selector: + matchLabels: + app.kubernetes.io/name: redis + template: + metadata: + labels: + app.kubernetes.io/name: redis + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: redis + image: redis:8.10.3-alpine + imagePullPolicy: IfNotPresent + command: + - sh + - /etc/redis/start-redis.sh + env: + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: redis-auth + key: username + - name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: redis-auth + key: password + ports: + - name: redis + containerPort: 6379 + startupProbe: + exec: + command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG'] + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 120 + readinessProbe: + exec: + command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG'] + periodSeconds: 5 + timeoutSeconds: 3 + livenessProbe: + exec: + command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG'] + periodSeconds: 10 + timeoutSeconds: 3 + resources: + requests: + cpu: 100m + memory: 4Gi + limits: + memory: 5Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + volumeMounts: + - name: data + mountPath: /data + - name: config + mountPath: /etc/redis + readOnly: true + - name: acl + mountPath: /run/redis-acl + - name: exporter + image: oliver006/redis_exporter:v1.94.0 + args: + - --redis.addr=redis://127.0.0.1:6379 + - --redis-only-metrics + - --disable-scrape-endpoint + - --connection-timeout=5s + env: + - name: REDIS_USER + valueFrom: + secretKeyRef: + name: redis-auth + key: username + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-auth + key: password + ports: + - name: metrics + containerPort: 9121 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumes: + - name: acl + emptyDir: + medium: Memory + sizeLimit: 1Mi + - name: config + configMap: + name: redis-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi diff --git a/platform/redis/overlays/main/infisical-secret.yaml b/platform/redis/overlays/main/infisical-secret.yaml new file mode 100644 index 0000000..0fa4fff --- /dev/null +++ b/platform/redis/overlays/main/infisical-secret.yaml @@ -0,0 +1,27 @@ +apiVersion: secrets.infisical.com/v1beta1 +kind: InfisicalStaticSecret +metadata: + name: redis-auth +spec: + infisicalAuthRef: + name: infisical-auth + namespace: infisical + syncOptions: + refreshInterval: 60s + instantUpdates: false + sources: + - projectId: 8520b8c0-3796-4fd6-a03c-6f0508c8c42c + environmentSlug: prod + secretPath: /platform/redis + recursive: false + targets: + - name: redis-auth + namespace: platform + kind: Secret + creationPolicy: Owner + secretType: Opaque + template: + engineVersion: v1 + data: + username: "{{ .REDIS_USERNAME.Value }}" + password: "{{ .REDIS_PASSWORD.Value }}" diff --git a/platform/redis/overlays/main/kustomization.yaml b/platform/redis/overlays/main/kustomization.yaml new file mode 100644 index 0000000..f5eeeb4 --- /dev/null +++ b/platform/redis/overlays/main/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: platform + +resources: + - ../../base + - infisical-secret.yaml + - monitoring.yaml + +patches: + - path: statefulset-patch.yaml diff --git a/platform/redis/overlays/main/monitoring.yaml b/platform/redis/overlays/main/monitoring.yaml new file mode 100644 index 0000000..7980f84 --- /dev/null +++ b/platform/redis/overlays/main/monitoring.yaml @@ -0,0 +1,147 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: redis + labels: + release: monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/name: redis + app.kubernetes.io/component: metrics + endpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s + relabelings: + - targetLabel: job + replacement: redis +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: redis + labels: + release: monitoring +spec: + groups: + - name: redis + rules: + - alert: RedisUnavailable + expr: | + redis_up{namespace="platform",job="redis"} == 0 + or up{namespace="platform",job="redis"} == 0 + or absent(redis_up{namespace="platform",job="redis"}) + for: 2m + labels: + severity: critical + annotations: + summary: Redis or its metrics endpoint is unavailable + description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target. + - alert: RedisMetricsScrapeFailed + expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0 + for: 2m + labels: + severity: warning + annotations: + summary: Redis metrics collection is failing + description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics. + - alert: RedisAofWriteFailed + expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Redis cannot persist writes to AOF + description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure. + - alert: RedisAofRewriteFailed + expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0 + for: 5m + labels: + severity: warning + annotations: + summary: Redis AOF rewrite failed + description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds. + - alert: RedisWriteRejected + expr: | + increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0 + or ( + redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0 + unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m + ) + labels: + severity: critical + annotations: + summary: Redis rejected writes due to memory or persistence errors + description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly. + - alert: RedisMemoryNearMaxmemory + expr: | + ( + redis_memory_used_bytes{namespace="platform",job="redis"} + - redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"} + ) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85 + for: 5m + labels: + severity: warning + annotations: + summary: Redis is approaching its maxmemory threshold + description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold. + - alert: RedisContainerMemoryHigh + expr: | + max by (namespace, pod, container) ( + container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"} + ) + / max by (namespace, pod, container) ( + kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"} + ) > 0.85 + for: 1m + labels: + severity: warning + annotations: + summary: Redis container is approaching its memory limit + description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting. + - alert: RedisContainerOomKilled + expr: | + max by (namespace, pod, container) ( + kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1 + ) + and on (namespace, pod, container) + ( + max by (namespace, pod, container) ( + increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m]) + ) > 0 + ) + labels: + severity: critical + annotations: + summary: Redis was restarted after an OOM kill + description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks. + - alert: RedisAofFsyncDelayed + expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0 + labels: + severity: warning + annotations: + summary: Redis AOF fsync is delayed + description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded. + - alert: RedisUnexpectedEviction + expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0 + labels: + severity: critical + annotations: + summary: Redis evicted keys despite the intended noeviction policy + description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners. + - alert: RedisNodeDiskHeadroomLow + # local-path currently stores Redis on heize's root filesystem. + # If the storage path moves to a dedicated mount, change mountpoint. + expr: | + ( + node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"} + * on (instance) group_left (nodename) + node_uname_info{job="node-exporter",nodename="heize"} + ) < 10 * 1024 * 1024 * 1024 + for: 5m + labels: + severity: warning + annotations: + summary: heize has less than 10GiB of free space for Redis AOF rewrites + description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space. diff --git a/platform/redis/overlays/main/statefulset-patch.yaml b/platform/redis/overlays/main/statefulset-patch.yaml new file mode 100644 index 0000000..ad15d9b --- /dev/null +++ b/platform/redis/overlays/main/statefulset-patch.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis +spec: + template: + metadata: + labels: + app.kubernetes.io/environment: prod + spec: + nodeSelector: + kubernetes.io/hostname: heize diff --git a/platform/redis/tests/check_monitoring.py b/platform/redis/tests/check_monitoring.py new file mode 100644 index 0000000..11e182a --- /dev/null +++ b/platform/redis/tests/check_monitoring.py @@ -0,0 +1,173 @@ +"""Validate rendered Redis resources and exercise alert behavior with promtool. + +Requires kubectl and promtool on PATH (or --promtool /path/to/promtool). +Uses client dry-run only; never applies resources to the cluster. +""" + +import argparse +import json +from pathlib import Path +import subprocess +import tempfile + + +def parse_json_stream(text): + decoder = json.JSONDecoder() + documents = [] + while text.strip(): + text = text.lstrip() + document, end = decoder.raw_decode(text) + documents.append(document) + text = text[end:] + return documents + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--promtool", default="promtool") + args = parser.parse_args() + app = Path(__file__).resolve().parents[1] + result = subprocess.run( + ["kubectl", "create", "--dry-run=client", "-k", + str(app / "overlays/main"), "-o", "json", "--request-timeout=10s"], + check=True, capture_output=True, text=True, encoding="utf-8", + ) + resources = parse_json_stream(result.stdout) + sts = next(r for r in resources if r["kind"] == "StatefulSet") + containers = sts["spec"]["template"]["spec"]["containers"] + redis = next(c for c in containers if c["name"] == "redis") + assert redis["resources"]["limits"]["memory"] == "5Gi" + assert redis["resources"]["requests"]["memory"] == "4Gi" + assert all("cpu" not in c["resources"].get("limits", {}) for c in containers) + config = next(r for r in resources if r["kind"] == "ConfigMap") + assert "maxmemory 3gb\n" in config["data"]["redis.conf"] + assert "maxmemory-policy noeviction\n" in config["data"]["redis.conf"] + metrics_service = next( + r for r in resources if r["kind"] == "Service" + and r["metadata"]["name"] == "redis-metrics" + ) + assert metrics_service["spec"]["publishNotReadyAddresses"] is True + monitor = next(r for r in resources if r["kind"] == "ServiceMonitor") + rule_resource = next(r for r in resources if r["kind"] == "PrometheusRule") + assert monitor["metadata"]["labels"]["release"] == "monitoring" + assert rule_resource["metadata"]["labels"]["release"] == "monitoring" + assert any( + relabel.get("targetLabel") == "job" and relabel.get("replacement") == "redis" + for relabel in monitor["spec"]["endpoints"][0]["relabelings"] + ) + rules = { + rule["alert"]: rule + for group in rule_resource["spec"]["groups"] + for rule in group["rules"] + } + base = {"namespace": "platform", "job": "redis"} + container = {"namespace": "platform", "pod": "redis-0", "container": "redis"} + + def series(metric, values, labels=None): + labels = base if labels is None else labels + selector = ",".join(f'{key}="{value}"' for key, value in labels.items()) + return {"series": f"{metric}{{{selector}}}", "values": values} + + tests = [] + + def case(name, alert, inputs, firing, labels=None, at="6m", healthy=True): + labels = base if labels is None else labels + expected = [] + if firing: + expected = [{ + "exp_labels": {**labels, **rules[alert]["labels"]}, + "exp_annotations": rules[alert]["annotations"], + }] + tests.append({ + "name": name, "interval": "30s", + "input_series": ( + [series("redis_up", "1+0x30")] if healthy else [] + ) + inputs, + "alert_rule_test": [{ + "eval_time": at, "alertname": alert, "exp_alerts": expected, + }], + }) + + case("healthy Redis", "RedisUnavailable", [], False) + case("missing exporter target", "RedisUnavailable", [], True, healthy=False) + case("Redis unreachable", "RedisUnavailable", + [series("redis_up", "0+0x30")], True, healthy=False) + case("HTTP endpoint down while Redis metric still exists", "RedisUnavailable", + [series("up", "0+0x30")], True) + case("brief outage is below hold duration", "RedisUnavailable", + [series("redis_up", "0+0x30")], False, at="1m", healthy=False) + case("exporter cannot read Redis INFO", "RedisMetricsScrapeFailed", + [series("redis_exporter_last_scrape_error", "1+0x30", {**base, "err": "NOPERM"})], + True, {**base, "err": "NOPERM"}) + for metric, alert in [ + ("redis_aof_last_write_status", "RedisAofWriteFailed"), + ("redis_aof_last_bgrewrite_status", "RedisAofRewriteFailed"), + ]: + case(f"{metric} ok is one", alert, [series(metric, "1+0x30")], False) + case(f"{metric} failure is zero", alert, [series(metric, "0+0x30")], True) + for error in ["OOM", "MISCONF"]: + labels = {**base, "err": error} + case(f"new {error} counter starts at one", "RedisWriteRejected", + [series("redis_errors_total", "_ _ _ _ 1+0x26", labels)], + True, labels, at="3m") + case(f"old {error} error does not alert forever", "RedisWriteRejected", + [series("redis_errors_total", "1+0x30", labels)], + False, at="10m") + case("growing OOM error counter", "RedisWriteRejected", + [series("redis_errors_total", "0+1x30", {**base, "err": "OOM"})], + True, {**base, "err": "OOM"}) + case("Redis memory high", "RedisMemoryNearMaxmemory", [ + series("redis_memory_used_bytes", "3006477107+0x30"), + series("redis_mem_not_counted_for_eviction_bytes", "0+0x30"), + series("redis_memory_max_bytes", "3221225472+0x30"), + ], True) + case("exclude buffers not counted by eviction", "RedisMemoryNearMaxmemory", [ + series("redis_memory_used_bytes", "3221225472+0x30"), + series("redis_mem_not_counted_for_eviction_bytes", "1073741824+0x30"), + series("redis_memory_max_bytes", "3221225472+0x30"), + ], False) + case("container limit evaluated independently from maxmemory", "RedisContainerMemoryHigh", [ + series("container_memory_working_set_bytes", "4831838208+0x30", container), + series("kube_pod_container_resource_limits", "5368709120+0x30", + {**container, "resource": "memory"}), + ], True, container) + for restarts, firing in [("0+1x30", True), ("1+0x30", False)]: + case(f"OOM reason with restarts {restarts}", "RedisContainerOomKilled", [ + series("kube_pod_container_status_last_terminated_reason", "1+0x30", + {**container, "reason": "OOMKilled"}), + series("kube_pod_container_status_restarts_total", restarts, container), + ], firing, container, at="12m") + for metric, alert in [ + ("redis_aof_delayed_fsync", "RedisAofFsyncDelayed"), + ("redis_evicted_keys_total", "RedisUnexpectedEviction"), + ]: + case(f"increasing {metric}", alert, [series(metric, "0+1x30")], True) + case(f"stable {metric}", alert, [series(metric, "1+0x30")], False) + disk = {"job": "node-exporter", "instance": "10.250.0.16:9100", + "mountpoint": "/", "fstype": "ext4", "device": "/dev/sda1"} + for node, free, firing in [ + ("heize", "9663676416", True), + ("heize", "12884901888", False), + ("suga", "9663676416", False), + ]: + case(f"{node} free disk {free}", "RedisNodeDiskHeadroomLow", [ + series("node_filesystem_avail_bytes", free + "+0x30", disk), + series("node_uname_info", "1+0x30", + {"job": "node-exporter", "instance": disk["instance"], "nodename": node}), + ], firing, {**disk, "nodename": node}) + + with tempfile.TemporaryDirectory(prefix="redis-monitoring-") as directory: + rule_path = Path(directory) / "rules.json" + rule_path.write_text(json.dumps(rule_resource["spec"]), encoding="utf-8") + test_path = Path(directory) / "tests.json" + test_path.write_text(json.dumps({ + "rule_files": [rule_path.as_posix()], + "evaluation_interval": "30s", "tests": tests, + }), encoding="utf-8") + subprocess.run([args.promtool, "check", "rules", str(rule_path)], check=True) + subprocess.run([args.promtool, "test", "rules", str(test_path)], check=True) + print(f"PASS: resource assertions and {len(tests)} alert scenarios") + + +if __name__ == "__main__": + main()