feat(redis): add shared Redis 8.10.3 to platform

This commit is contained in:
dahyeon committed 2026-11-09 16:01:48 +09:00
1 parent eb96f56349
commit 67356b1489
13 files changed
+671

No files matched your search

+173
View File
@@ -0,0 +1,173 @@
"""Validate rendered Redis resources and exercise alert behavior with promtool.
Requires kubectl and promtool on PATH (or --promtool /path/to/promtool).
Uses client dry-run only; never applies resources to the cluster.
"""
import argparse
import json
from pathlib import Path
import subprocess
import tempfile
def parse_json_stream(text):
decoder = json.JSONDecoder()
documents = []
while text.strip():
text = text.lstrip()
document, end = decoder.raw_decode(text)
documents.append(document)
text = text[end:]
return documents
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--promtool", default="promtool")
args = parser.parse_args()
app = Path(__file__).resolve().parents[1]
result = subprocess.run(
["kubectl", "create", "--dry-run=client", "-k",
str(app / "overlays/main"), "-o", "json", "--request-timeout=10s"],
check=True, capture_output=True, text=True, encoding="utf-8",
)
resources = parse_json_stream(result.stdout)
sts = next(r for r in resources if r["kind"] == "StatefulSet")
containers = sts["spec"]["template"]["spec"]["containers"]
redis = next(c for c in containers if c["name"] == "redis")
assert redis["resources"]["limits"]["memory"] == "5Gi"
assert redis["resources"]["requests"]["memory"] == "4Gi"
assert all("cpu" not in c["resources"].get("limits", {}) for c in containers)
config = next(r for r in resources if r["kind"] == "ConfigMap")
assert "maxmemory 3gb\n" in config["data"]["redis.conf"]
assert "maxmemory-policy noeviction\n" in config["data"]["redis.conf"]
metrics_service = next(
r for r in resources if r["kind"] == "Service"
and r["metadata"]["name"] == "redis-metrics"
)
assert metrics_service["spec"]["publishNotReadyAddresses"] is True
monitor = next(r for r in resources if r["kind"] == "ServiceMonitor")
rule_resource = next(r for r in resources if r["kind"] == "PrometheusRule")
assert monitor["metadata"]["labels"]["release"] == "monitoring"
assert rule_resource["metadata"]["labels"]["release"] == "monitoring"
assert any(
relabel.get("targetLabel") == "job" and relabel.get("replacement") == "redis"
for relabel in monitor["spec"]["endpoints"][0]["relabelings"]
)
rules = {
rule["alert"]: rule
for group in rule_resource["spec"]["groups"]
for rule in group["rules"]
}
base = {"namespace": "platform", "job": "redis"}
container = {"namespace": "platform", "pod": "redis-0", "container": "redis"}
def series(metric, values, labels=None):
labels = base if labels is None else labels
selector = ",".join(f'{key}="{value}"' for key, value in labels.items())
return {"series": f"{metric}{{{selector}}}", "values": values}
tests = []
def case(name, alert, inputs, firing, labels=None, at="6m", healthy=True):
labels = base if labels is None else labels
expected = []
if firing:
expected = [{
"exp_labels": {**labels, **rules[alert]["labels"]},
"exp_annotations": rules[alert]["annotations"],
}]
tests.append({
"name": name, "interval": "30s",
"input_series": (
[series("redis_up", "1+0x30")] if healthy else []
) + inputs,
"alert_rule_test": [{
"eval_time": at, "alertname": alert, "exp_alerts": expected,
}],
})
case("healthy Redis", "RedisUnavailable", [], False)
case("missing exporter target", "RedisUnavailable", [], True, healthy=False)
case("Redis unreachable", "RedisUnavailable",
[series("redis_up", "0+0x30")], True, healthy=False)
case("HTTP endpoint down while Redis metric still exists", "RedisUnavailable",
[series("up", "0+0x30")], True)
case("brief outage is below hold duration", "RedisUnavailable",
[series("redis_up", "0+0x30")], False, at="1m", healthy=False)
case("exporter cannot read Redis INFO", "RedisMetricsScrapeFailed",
[series("redis_exporter_last_scrape_error", "1+0x30", {**base, "err": "NOPERM"})],
True, {**base, "err": "NOPERM"})
for metric, alert in [
("redis_aof_last_write_status", "RedisAofWriteFailed"),
("redis_aof_last_bgrewrite_status", "RedisAofRewriteFailed"),
]:
case(f"{metric} ok is one", alert, [series(metric, "1+0x30")], False)
case(f"{metric} failure is zero", alert, [series(metric, "0+0x30")], True)
for error in ["OOM", "MISCONF"]:
labels = {**base, "err": error}
case(f"new {error} counter starts at one", "RedisWriteRejected",
[series("redis_errors_total", "_ _ _ _ 1+0x26", labels)],
True, labels, at="3m")
case(f"old {error} error does not alert forever", "RedisWriteRejected",
[series("redis_errors_total", "1+0x30", labels)],
False, at="10m")
case("growing OOM error counter", "RedisWriteRejected",
[series("redis_errors_total", "0+1x30", {**base, "err": "OOM"})],
True, {**base, "err": "OOM"})
case("Redis memory high", "RedisMemoryNearMaxmemory", [
series("redis_memory_used_bytes", "3006477107+0x30"),
series("redis_mem_not_counted_for_eviction_bytes", "0+0x30"),
series("redis_memory_max_bytes", "3221225472+0x30"),
], True)
case("exclude buffers not counted by eviction", "RedisMemoryNearMaxmemory", [
series("redis_memory_used_bytes", "3221225472+0x30"),
series("redis_mem_not_counted_for_eviction_bytes", "1073741824+0x30"),
series("redis_memory_max_bytes", "3221225472+0x30"),
], False)
case("container limit evaluated independently from maxmemory", "RedisContainerMemoryHigh", [
series("container_memory_working_set_bytes", "4831838208+0x30", container),
series("kube_pod_container_resource_limits", "5368709120+0x30",
{**container, "resource": "memory"}),
], True, container)
for restarts, firing in [("0+1x30", True), ("1+0x30", False)]:
case(f"OOM reason with restarts {restarts}", "RedisContainerOomKilled", [
series("kube_pod_container_status_last_terminated_reason", "1+0x30",
{**container, "reason": "OOMKilled"}),
series("kube_pod_container_status_restarts_total", restarts, container),
], firing, container, at="12m")
for metric, alert in [
("redis_aof_delayed_fsync", "RedisAofFsyncDelayed"),
("redis_evicted_keys_total", "RedisUnexpectedEviction"),
]:
case(f"increasing {metric}", alert, [series(metric, "0+1x30")], True)
case(f"stable {metric}", alert, [series(metric, "1+0x30")], False)
disk = {"job": "node-exporter", "instance": "10.250.0.16:9100",
"mountpoint": "/", "fstype": "ext4", "device": "/dev/sda1"}
for node, free, firing in [
("heize", "9663676416", True),
("heize", "12884901888", False),
("suga", "9663676416", False),
]:
case(f"{node} free disk {free}", "RedisNodeDiskHeadroomLow", [
series("node_filesystem_avail_bytes", free + "+0x30", disk),
series("node_uname_info", "1+0x30",
{"job": "node-exporter", "instance": disk["instance"], "nodename": node}),
], firing, {**disk, "nodename": node})
with tempfile.TemporaryDirectory(prefix="redis-monitoring-") as directory:
rule_path = Path(directory) / "rules.json"
rule_path.write_text(json.dumps(rule_resource["spec"]), encoding="utf-8")
test_path = Path(directory) / "tests.json"
test_path.write_text(json.dumps({
"rule_files": [rule_path.as_posix()],
"evaluation_interval": "30s", "tests": tests,
}), encoding="utf-8")
subprocess.run([args.promtool, "check", "rules", str(rule_path)], check=True)
subprocess.run([args.promtool, "test", "rules", str(test_path)], check=True)
print(f"PASS: resource assertions and {len(tests)} alert scenarios")
if __name__ == "__main__":
main()