"""Validate rendered Redis resources and exercise alert behavior with promtool. Requires kubectl and promtool on PATH (or --promtool /path/to/promtool). Uses client dry-run only; never applies resources to the cluster. """ import argparse import json from pathlib import Path import subprocess import tempfile def parse_json_stream(text): decoder = json.JSONDecoder() documents = [] while text.strip(): text = text.lstrip() document, end = decoder.raw_decode(text) documents.append(document) text = text[end:] return documents def main(): parser = argparse.ArgumentParser() parser.add_argument("--promtool", default="promtool") args = parser.parse_args() app = Path(__file__).resolve().parents[1] result = subprocess.run( ["kubectl", "create", "--dry-run=client", "-k", str(app / "overlays/main"), "-o", "json", "--request-timeout=10s"], check=True, capture_output=True, text=True, encoding="utf-8", ) resources = parse_json_stream(result.stdout) sts = next(r for r in resources if r["kind"] == "StatefulSet") containers = sts["spec"]["template"]["spec"]["containers"] redis = next(c for c in containers if c["name"] == "redis") assert redis["resources"]["limits"]["memory"] == "5Gi" assert redis["resources"]["requests"]["memory"] == "4Gi" assert all("cpu" not in c["resources"].get("limits", {}) for c in containers) config = next(r for r in resources if r["kind"] == "ConfigMap") assert "maxmemory 3gb\n" in config["data"]["redis.conf"] assert "maxmemory-policy noeviction\n" in config["data"]["redis.conf"] metrics_service = next( r for r in resources if r["kind"] == "Service" and r["metadata"]["name"] == "redis-metrics" ) assert metrics_service["spec"]["publishNotReadyAddresses"] is True monitor = next(r for r in resources if r["kind"] == "ServiceMonitor") rule_resource = next(r for r in resources if r["kind"] == "PrometheusRule") assert monitor["metadata"]["labels"]["release"] == "monitoring" assert rule_resource["metadata"]["labels"]["release"] == "monitoring" assert any( relabel.get("targetLabel") == "job" and relabel.get("replacement") == "redis" for relabel in monitor["spec"]["endpoints"][0]["relabelings"] ) rules = { rule["alert"]: rule for group in rule_resource["spec"]["groups"] for rule in group["rules"] } base = {"namespace": "platform", "job": "redis"} container = {"namespace": "platform", "pod": "redis-0", "container": "redis"} def series(metric, values, labels=None): labels = base if labels is None else labels selector = ",".join(f'{key}="{value}"' for key, value in labels.items()) return {"series": f"{metric}{{{selector}}}", "values": values} tests = [] def case(name, alert, inputs, firing, labels=None, at="6m", healthy=True): labels = base if labels is None else labels expected = [] if firing: expected = [{ "exp_labels": {**labels, **rules[alert]["labels"]}, "exp_annotations": rules[alert]["annotations"], }] tests.append({ "name": name, "interval": "30s", "input_series": ( [series("redis_up", "1+0x30")] if healthy else [] ) + inputs, "alert_rule_test": [{ "eval_time": at, "alertname": alert, "exp_alerts": expected, }], }) case("healthy Redis", "RedisUnavailable", [], False) case("missing exporter target", "RedisUnavailable", [], True, healthy=False) case("Redis unreachable", "RedisUnavailable", [series("redis_up", "0+0x30")], True, healthy=False) case("HTTP endpoint down while Redis metric still exists", "RedisUnavailable", [series("up", "0+0x30")], True) case("brief outage is below hold duration", "RedisUnavailable", [series("redis_up", "0+0x30")], False, at="1m", healthy=False) case("exporter cannot read Redis INFO", "RedisMetricsScrapeFailed", [series("redis_exporter_last_scrape_error", "1+0x30", {**base, "err": "NOPERM"})], True, {**base, "err": "NOPERM"}) for metric, alert in [ ("redis_aof_last_write_status", "RedisAofWriteFailed"), ("redis_aof_last_bgrewrite_status", "RedisAofRewriteFailed"), ]: case(f"{metric} ok is one", alert, [series(metric, "1+0x30")], False) case(f"{metric} failure is zero", alert, [series(metric, "0+0x30")], True) for error in ["OOM", "MISCONF"]: labels = {**base, "err": error} case(f"new {error} counter starts at one", "RedisWriteRejected", [series("redis_errors_total", "_ _ _ _ 1+0x26", labels)], True, labels, at="3m") case(f"old {error} error does not alert forever", "RedisWriteRejected", [series("redis_errors_total", "1+0x30", labels)], False, at="10m") case("growing OOM error counter", "RedisWriteRejected", [series("redis_errors_total", "0+1x30", {**base, "err": "OOM"})], True, {**base, "err": "OOM"}) case("Redis memory high", "RedisMemoryNearMaxmemory", [ series("redis_memory_used_bytes", "3006477107+0x30"), series("redis_mem_not_counted_for_eviction_bytes", "0+0x30"), series("redis_memory_max_bytes", "3221225472+0x30"), ], True) case("exclude buffers not counted by eviction", "RedisMemoryNearMaxmemory", [ series("redis_memory_used_bytes", "3221225472+0x30"), series("redis_mem_not_counted_for_eviction_bytes", "1073741824+0x30"), series("redis_memory_max_bytes", "3221225472+0x30"), ], False) case("container limit evaluated independently from maxmemory", "RedisContainerMemoryHigh", [ series("container_memory_working_set_bytes", "4831838208+0x30", container), series("kube_pod_container_resource_limits", "5368709120+0x30", {**container, "resource": "memory"}), ], True, container) for restarts, firing in [("0+1x30", True), ("1+0x30", False)]: case(f"OOM reason with restarts {restarts}", "RedisContainerOomKilled", [ series("kube_pod_container_status_last_terminated_reason", "1+0x30", {**container, "reason": "OOMKilled"}), series("kube_pod_container_status_restarts_total", restarts, container), ], firing, container, at="12m") for metric, alert in [ ("redis_aof_delayed_fsync", "RedisAofFsyncDelayed"), ("redis_evicted_keys_total", "RedisUnexpectedEviction"), ]: case(f"increasing {metric}", alert, [series(metric, "0+1x30")], True) case(f"stable {metric}", alert, [series(metric, "1+0x30")], False) disk = {"job": "node-exporter", "instance": "10.250.0.16:9100", "mountpoint": "/", "fstype": "ext4", "device": "/dev/sda1"} for node, free, firing in [ ("heize", "9663676416", True), ("heize", "12884901888", False), ("suga", "9663676416", False), ]: case(f"{node} free disk {free}", "RedisNodeDiskHeadroomLow", [ series("node_filesystem_avail_bytes", free + "+0x30", disk), series("node_uname_info", "1+0x30", {"job": "node-exporter", "instance": disk["instance"], "nodename": node}), ], firing, {**disk, "nodename": node}) with tempfile.TemporaryDirectory(prefix="redis-monitoring-") as directory: rule_path = Path(directory) / "rules.json" rule_path.write_text(json.dumps(rule_resource["spec"]), encoding="utf-8") test_path = Path(directory) / "tests.json" test_path.write_text(json.dumps({ "rule_files": [rule_path.as_posix()], "evaluation_interval": "30s", "tests": tests, }), encoding="utf-8") subprocess.run([args.promtool, "check", "rules", str(rule_path)], check=True) subprocess.run([args.promtool, "test", "rules", str(test_path)], check=True) print(f"PASS: resource assertions and {len(tests)} alert scenarios") if __name__ == "__main__": main()