feat(redis): add shared Redis 8.10.3 to platform
This commit is contained in:
1 parent
eb96f56349
commit
67356b1489
13 files changed
+671
No files matched your search
@@ -0,0 +1,173 @@
|
||||
"""Validate rendered Redis resources and exercise alert behavior with promtool.
|
||||
|
||||
Requires kubectl and promtool on PATH (or --promtool /path/to/promtool).
|
||||
Uses client dry-run only; never applies resources to the cluster.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
def parse_json_stream(text):
|
||||
decoder = json.JSONDecoder()
|
||||
documents = []
|
||||
while text.strip():
|
||||
text = text.lstrip()
|
||||
document, end = decoder.raw_decode(text)
|
||||
documents.append(document)
|
||||
text = text[end:]
|
||||
return documents
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--promtool", default="promtool")
|
||||
args = parser.parse_args()
|
||||
app = Path(__file__).resolve().parents[1]
|
||||
result = subprocess.run(
|
||||
["kubectl", "create", "--dry-run=client", "-k",
|
||||
str(app / "overlays/main"), "-o", "json", "--request-timeout=10s"],
|
||||
check=True, capture_output=True, text=True, encoding="utf-8",
|
||||
)
|
||||
resources = parse_json_stream(result.stdout)
|
||||
sts = next(r for r in resources if r["kind"] == "StatefulSet")
|
||||
containers = sts["spec"]["template"]["spec"]["containers"]
|
||||
redis = next(c for c in containers if c["name"] == "redis")
|
||||
assert redis["resources"]["limits"]["memory"] == "5Gi"
|
||||
assert redis["resources"]["requests"]["memory"] == "4Gi"
|
||||
assert all("cpu" not in c["resources"].get("limits", {}) for c in containers)
|
||||
config = next(r for r in resources if r["kind"] == "ConfigMap")
|
||||
assert "maxmemory 3gb\n" in config["data"]["redis.conf"]
|
||||
assert "maxmemory-policy noeviction\n" in config["data"]["redis.conf"]
|
||||
metrics_service = next(
|
||||
r for r in resources if r["kind"] == "Service"
|
||||
and r["metadata"]["name"] == "redis-metrics"
|
||||
)
|
||||
assert metrics_service["spec"]["publishNotReadyAddresses"] is True
|
||||
monitor = next(r for r in resources if r["kind"] == "ServiceMonitor")
|
||||
rule_resource = next(r for r in resources if r["kind"] == "PrometheusRule")
|
||||
assert monitor["metadata"]["labels"]["release"] == "monitoring"
|
||||
assert rule_resource["metadata"]["labels"]["release"] == "monitoring"
|
||||
assert any(
|
||||
relabel.get("targetLabel") == "job" and relabel.get("replacement") == "redis"
|
||||
for relabel in monitor["spec"]["endpoints"][0]["relabelings"]
|
||||
)
|
||||
rules = {
|
||||
rule["alert"]: rule
|
||||
for group in rule_resource["spec"]["groups"]
|
||||
for rule in group["rules"]
|
||||
}
|
||||
base = {"namespace": "platform", "job": "redis"}
|
||||
container = {"namespace": "platform", "pod": "redis-0", "container": "redis"}
|
||||
|
||||
def series(metric, values, labels=None):
|
||||
labels = base if labels is None else labels
|
||||
selector = ",".join(f'{key}="{value}"' for key, value in labels.items())
|
||||
return {"series": f"{metric}{{{selector}}}", "values": values}
|
||||
|
||||
tests = []
|
||||
|
||||
def case(name, alert, inputs, firing, labels=None, at="6m", healthy=True):
|
||||
labels = base if labels is None else labels
|
||||
expected = []
|
||||
if firing:
|
||||
expected = [{
|
||||
"exp_labels": {**labels, **rules[alert]["labels"]},
|
||||
"exp_annotations": rules[alert]["annotations"],
|
||||
}]
|
||||
tests.append({
|
||||
"name": name, "interval": "30s",
|
||||
"input_series": (
|
||||
[series("redis_up", "1+0x30")] if healthy else []
|
||||
) + inputs,
|
||||
"alert_rule_test": [{
|
||||
"eval_time": at, "alertname": alert, "exp_alerts": expected,
|
||||
}],
|
||||
})
|
||||
|
||||
case("healthy Redis", "RedisUnavailable", [], False)
|
||||
case("missing exporter target", "RedisUnavailable", [], True, healthy=False)
|
||||
case("Redis unreachable", "RedisUnavailable",
|
||||
[series("redis_up", "0+0x30")], True, healthy=False)
|
||||
case("HTTP endpoint down while Redis metric still exists", "RedisUnavailable",
|
||||
[series("up", "0+0x30")], True)
|
||||
case("brief outage is below hold duration", "RedisUnavailable",
|
||||
[series("redis_up", "0+0x30")], False, at="1m", healthy=False)
|
||||
case("exporter cannot read Redis INFO", "RedisMetricsScrapeFailed",
|
||||
[series("redis_exporter_last_scrape_error", "1+0x30", {**base, "err": "NOPERM"})],
|
||||
True, {**base, "err": "NOPERM"})
|
||||
for metric, alert in [
|
||||
("redis_aof_last_write_status", "RedisAofWriteFailed"),
|
||||
("redis_aof_last_bgrewrite_status", "RedisAofRewriteFailed"),
|
||||
]:
|
||||
case(f"{metric} ok is one", alert, [series(metric, "1+0x30")], False)
|
||||
case(f"{metric} failure is zero", alert, [series(metric, "0+0x30")], True)
|
||||
for error in ["OOM", "MISCONF"]:
|
||||
labels = {**base, "err": error}
|
||||
case(f"new {error} counter starts at one", "RedisWriteRejected",
|
||||
[series("redis_errors_total", "_ _ _ _ 1+0x26", labels)],
|
||||
True, labels, at="3m")
|
||||
case(f"old {error} error does not alert forever", "RedisWriteRejected",
|
||||
[series("redis_errors_total", "1+0x30", labels)],
|
||||
False, at="10m")
|
||||
case("growing OOM error counter", "RedisWriteRejected",
|
||||
[series("redis_errors_total", "0+1x30", {**base, "err": "OOM"})],
|
||||
True, {**base, "err": "OOM"})
|
||||
case("Redis memory high", "RedisMemoryNearMaxmemory", [
|
||||
series("redis_memory_used_bytes", "3006477107+0x30"),
|
||||
series("redis_mem_not_counted_for_eviction_bytes", "0+0x30"),
|
||||
series("redis_memory_max_bytes", "3221225472+0x30"),
|
||||
], True)
|
||||
case("exclude buffers not counted by eviction", "RedisMemoryNearMaxmemory", [
|
||||
series("redis_memory_used_bytes", "3221225472+0x30"),
|
||||
series("redis_mem_not_counted_for_eviction_bytes", "1073741824+0x30"),
|
||||
series("redis_memory_max_bytes", "3221225472+0x30"),
|
||||
], False)
|
||||
case("container limit evaluated independently from maxmemory", "RedisContainerMemoryHigh", [
|
||||
series("container_memory_working_set_bytes", "4831838208+0x30", container),
|
||||
series("kube_pod_container_resource_limits", "5368709120+0x30",
|
||||
{**container, "resource": "memory"}),
|
||||
], True, container)
|
||||
for restarts, firing in [("0+1x30", True), ("1+0x30", False)]:
|
||||
case(f"OOM reason with restarts {restarts}", "RedisContainerOomKilled", [
|
||||
series("kube_pod_container_status_last_terminated_reason", "1+0x30",
|
||||
{**container, "reason": "OOMKilled"}),
|
||||
series("kube_pod_container_status_restarts_total", restarts, container),
|
||||
], firing, container, at="12m")
|
||||
for metric, alert in [
|
||||
("redis_aof_delayed_fsync", "RedisAofFsyncDelayed"),
|
||||
("redis_evicted_keys_total", "RedisUnexpectedEviction"),
|
||||
]:
|
||||
case(f"increasing {metric}", alert, [series(metric, "0+1x30")], True)
|
||||
case(f"stable {metric}", alert, [series(metric, "1+0x30")], False)
|
||||
disk = {"job": "node-exporter", "instance": "10.250.0.16:9100",
|
||||
"mountpoint": "/", "fstype": "ext4", "device": "/dev/sda1"}
|
||||
for node, free, firing in [
|
||||
("heize", "9663676416", True),
|
||||
("heize", "12884901888", False),
|
||||
("suga", "9663676416", False),
|
||||
]:
|
||||
case(f"{node} free disk {free}", "RedisNodeDiskHeadroomLow", [
|
||||
series("node_filesystem_avail_bytes", free + "+0x30", disk),
|
||||
series("node_uname_info", "1+0x30",
|
||||
{"job": "node-exporter", "instance": disk["instance"], "nodename": node}),
|
||||
], firing, {**disk, "nodename": node})
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="redis-monitoring-") as directory:
|
||||
rule_path = Path(directory) / "rules.json"
|
||||
rule_path.write_text(json.dumps(rule_resource["spec"]), encoding="utf-8")
|
||||
test_path = Path(directory) / "tests.json"
|
||||
test_path.write_text(json.dumps({
|
||||
"rule_files": [rule_path.as_posix()],
|
||||
"evaluation_interval": "30s", "tests": tests,
|
||||
}), encoding="utf-8")
|
||||
subprocess.run([args.promtool, "check", "rules", str(rule_path)], check=True)
|
||||
subprocess.run([args.promtool, "test", "rules", str(test_path)], check=True)
|
||||
print(f"PASS: resource assertions and {len(tests)} alert scenarios")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user