Compare commits
12
Commits
c2e98ca321
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d6cbf1f23f | ||
|
|
ecfc86c72f | ||
|
|
083eae7f8c | ||
|
|
67356b1489 | ||
|
|
eb96f56349 | ||
|
|
2b65a583b9 | ||
|
|
1e20c435cd | ||
|
|
6df004a1da | ||
|
|
8b415c4e94 | ||
|
|
2d0df191dc | ||
|
|
b14c8e5b43 | ||
|
|
9ec1eebe2e |
No files matched your search
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: blog-ci
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
project: ci
|
||||||
|
source:
|
||||||
|
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
|
||||||
|
targetRevision: main
|
||||||
|
path: ci/blog
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: tekton-ci
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- SkipDryRunOnMissingResource=true
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: lolympus-tournament-support-ci
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
project: ci
|
||||||
|
source:
|
||||||
|
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
|
||||||
|
targetRevision: main
|
||||||
|
path: ci/lolympus-tournament-support
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: tekton-ci
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- SkipDryRunOnMissingResource=true
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: make-space-ci
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
project: ci
|
||||||
|
source:
|
||||||
|
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
|
||||||
|
targetRevision: main
|
||||||
|
path: ci/make-space
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: tekton-ci
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- SkipDryRunOnMissingResource=true
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "5"
|
||||||
|
spec:
|
||||||
|
project: platform
|
||||||
|
source:
|
||||||
|
repoURL: https://git.dahyeon.us/dahyeon/platform-state.git
|
||||||
|
targetRevision: main
|
||||||
|
path: platform/redis/overlays/main
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: platform
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: url-shortener-ci
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
project: ci
|
||||||
|
source:
|
||||||
|
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
|
||||||
|
targetRevision: main
|
||||||
|
path: ci/url-shortener
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: tekton-ci
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- SkipDryRunOnMissingResource=true
|
||||||
@@ -26,3 +26,13 @@ spec:
|
|||||||
destination:
|
destination:
|
||||||
server: https://kubernetes.default.svc
|
server: https://kubernetes.default.svc
|
||||||
namespace: "{{index .path.segments 1}}-prod"
|
namespace: "{{index .path.segments 1}}-prod"
|
||||||
|
templatePatch: |
|
||||||
|
{{- if or (eq (index .path.segments 1) "url-shortener") (eq (index .path.segments 1) "make-space") (eq (index .path.segments 1) "blog") }}
|
||||||
|
spec:
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
{{- else }}
|
||||||
|
{}
|
||||||
|
{{- end }}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: AppProject
|
||||||
|
metadata:
|
||||||
|
name: ci
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "-20"
|
||||||
|
spec:
|
||||||
|
description: Namespaced CI delivery resources from service-state
|
||||||
|
sourceRepos:
|
||||||
|
- https://git.dahyeon.us/dahyeon/service-state.git
|
||||||
|
destinations:
|
||||||
|
- server: https://kubernetes.default.svc
|
||||||
|
namespace: tekton-ci
|
||||||
|
namespaceResourceWhitelist:
|
||||||
|
- group: ""
|
||||||
|
kind: ServiceAccount
|
||||||
|
- group: rbac.authorization.k8s.io
|
||||||
|
kind: Role
|
||||||
|
- group: rbac.authorization.k8s.io
|
||||||
|
kind: RoleBinding
|
||||||
|
- group: networking.k8s.io
|
||||||
|
kind: Ingress
|
||||||
|
- group: secrets.infisical.com
|
||||||
|
kind: InfisicalStaticSecret
|
||||||
|
- group: tekton.dev
|
||||||
|
kind: Task
|
||||||
|
- group: tekton.dev
|
||||||
|
kind: Pipeline
|
||||||
|
- group: triggers.tekton.dev
|
||||||
|
kind: EventListener
|
||||||
|
- group: triggers.tekton.dev
|
||||||
|
kind: TriggerBinding
|
||||||
|
- group: triggers.tekton.dev
|
||||||
|
kind: TriggerTemplate
|
||||||
|
orphanedResources:
|
||||||
|
warn: true
|
||||||
@@ -87,3 +87,28 @@ apiVersion: v1
|
|||||||
kind: Namespace
|
kind: Namespace
|
||||||
metadata:
|
metadata:
|
||||||
name: persona-lenz-staging
|
name: persona-lenz-staging
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: make-space-staging
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: make-space-prod
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: blog-staging
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: blog-prod
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: platform
|
||||||
+88
-2
@@ -1,5 +1,11 @@
|
|||||||
# Spec: Tekton CI/CD for lolympus-tournament-support
|
# Spec: Tekton CI/CD for lolympus-tournament-support
|
||||||
|
|
||||||
|
This document's original main-to-staging/manual-production contract is specific
|
||||||
|
to lolympus. Blog follows make-space: source `staging` and `prod` branches each
|
||||||
|
run a dedicated pipeline and both environments auto-sync. Blog registration is
|
||||||
|
`argocd/applications/blog-ci.yaml`; its complete delivery/secret contract is in
|
||||||
|
service-state `ci/blog/README.md` and `apps/blog/README.md`.
|
||||||
|
|
||||||
## Objective
|
## Objective
|
||||||
|
|
||||||
Build a GitOps-native staging delivery path for the lolympus monorepo on the
|
Build a GitOps-native staging delivery path for the lolympus monorepo on the
|
||||||
@@ -87,6 +93,71 @@ service-state/
|
|||||||
Tekton controllers stay in `tekton-pipelines`. Application PipelineRuns and
|
Tekton controllers stay in `tekton-pipelines`. Application PipelineRuns and
|
||||||
their credentials run in the isolated `tekton-ci` namespace.
|
their credentials run in the isolated `tekton-ci` namespace.
|
||||||
|
|
||||||
|
The `tekton-ci` namespace is shared by all service CI applications. Its
|
||||||
|
ResourceQuota and LimitRange are owned by the `tekton-config` Argo CD Application
|
||||||
|
from `platform/tekton/config/ci-resource-policy.yaml`. Their legacy resource name
|
||||||
|
`lolympus-ci` is retained to transfer ownership without recreating either object;
|
||||||
|
it does not restrict the policies to the lolympus service. The service `ci`
|
||||||
|
AppProject does not permit ResourceQuota or LimitRange resources.
|
||||||
|
|
||||||
|
The shared quota permits at most 15 PipelineRuns, 15 PVCs requesting a total of
|
||||||
|
150 GiB, and 150 TaskRuns. The job-based pruner runs at minute 00, 15, 30, and 45
|
||||||
|
of every hour and retains the newest 10 completed PipelineRuns per namespace,
|
||||||
|
across pipeline names and success/failure outcomes. Running and pending runs are
|
||||||
|
protected and are additional to the retained completed history. Child TaskRuns,
|
||||||
|
Pods, and workspace PVCs follow their owning PipelineRun's deletion; TaskRuns
|
||||||
|
are not pruned independently. If 15 runs accumulate before pruning frees a slot,
|
||||||
|
the quota rejects further PipelineRun creation.
|
||||||
|
|
||||||
|
### Transferring the shared resource policies
|
||||||
|
|
||||||
|
This transfer spans two repositories with independent automatic synchronization.
|
||||||
|
Do not rely on Application sync waves to order commits across the repositories.
|
||||||
|
For an existing cluster where `lolympus-tournament-support-ci` owns the policies:
|
||||||
|
|
||||||
|
1. Wait for active sync operations to finish. Temporarily disable automated sync
|
||||||
|
on `platform-root`, `tekton-config`, and `lolympus-tournament-support-ci`,
|
||||||
|
preserving their existing sync policy for restoration.
|
||||||
|
2. Record the UID and tracking annotation of both existing `lolympus-ci` objects
|
||||||
|
in `tekton-ci`. Add the temporary resource annotation
|
||||||
|
`argocd.argoproj.io/sync-options: Prune=false` to both objects.
|
||||||
|
3. Publish the service-state removal and platform-state adoption commits. Sync
|
||||||
|
`lolympus-tournament-support-ci` to the removal commit first. The protected
|
||||||
|
policies must remain present even though they are no longer desired there.
|
||||||
|
4. Sync `tekton-config` to the adoption commit. Verify both objects retain their
|
||||||
|
UIDs and now have tracking IDs beginning with `tekton-config:`. Verify the
|
||||||
|
quota values and the generated pruner CronJob's schedule and keep arguments.
|
||||||
|
5. Refresh the old Application and confirm it no longer tracks either policy.
|
||||||
|
Remove only the temporary `Prune=false` annotation after adoption succeeds.
|
||||||
|
6. Restore automated sync on the child Applications, then sync `platform-root`
|
||||||
|
to apply the tightened `ci` AppProject permissions and restore its automation.
|
||||||
|
Confirm all three Applications are Synced and Healthy.
|
||||||
|
|
||||||
|
If adoption fails, leave the existing policies protected and avoid syncing the
|
||||||
|
new AppProject restrictions until ownership is resolved. Do not delete and
|
||||||
|
recreate the policies or remove unrelated sync-option annotations.
|
||||||
|
|
||||||
|
## Registering Additional Service CI
|
||||||
|
|
||||||
|
CI directories in `service-state` are registered explicitly; the service
|
||||||
|
ApplicationSets discover only `apps/*/overlays/*`.
|
||||||
|
|
||||||
|
For URL shortener, `argocd/applications/url-shortener-ci.yaml` reconciles
|
||||||
|
`service-state/ci/url-shortener` into `tekton-ci` using the `ci` project.
|
||||||
|
The platform root Application discovers this registration recursively.
|
||||||
|
|
||||||
|
Cluster-scoped EventListener RBAC belongs in
|
||||||
|
`platform/tekton/config/eventlistener-rbac.yaml`. Each listener receives a
|
||||||
|
dedicated ClusterRole and ClusterRoleBinding allowing only `get`, `list`, and
|
||||||
|
`watch` on `clusterinterceptors` and `clustertriggerbindings`.
|
||||||
|
Keep ServiceAccounts, Roles, and RoleBindings in `service-state`; remove the
|
||||||
|
URL shortener ClusterRoleBinding from its `foundation.yaml` when adopting this
|
||||||
|
registration. The `ci` project intentionally does not allow cluster resources.
|
||||||
|
|
||||||
|
Before the first URL shortener run, configure its Infisical secrets and Gitea
|
||||||
|
push webhook at `https://tekton-hook.dahyeon.us/url-shortener` as defined in
|
||||||
|
`service-state/ci/url-shortener`.
|
||||||
|
|
||||||
## Code Style
|
## Code Style
|
||||||
|
|
||||||
Use stable Tekton APIs, explicit names, immutable image references, bounded
|
Use stable Tekton APIs, explicit names, immutable image references, bounded
|
||||||
@@ -107,12 +178,15 @@ spec:
|
|||||||
script: |
|
script: |
|
||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
set -eu
|
set -eu
|
||||||
corepack pnpm --filter api test -- --runInBand
|
pnpm --filter api test --runInBand
|
||||||
```
|
```
|
||||||
|
|
||||||
Do not use mutable image tags such as `latest`, inline credentials, or direct
|
Do not use mutable image tags such as `latest`, inline credentials, or direct
|
||||||
`kubectl set image` deployment mutations.
|
`kubectl set image` deployment mutations.
|
||||||
|
|
||||||
|
The pinned Node 26 image does not include Corepack. The Tasks bootstrap exactly
|
||||||
|
`pnpm@10.4.1` into the shared workspace before running package scripts.
|
||||||
|
|
||||||
## Secret Contract
|
## Secret Contract
|
||||||
|
|
||||||
Infisical project `8520b8c0-3796-4fd6-a03c-6f0508c8c42c`, environment
|
Infisical project `8520b8c0-3796-4fd6-a03c-6f0508c8c42c`, environment
|
||||||
@@ -129,6 +203,18 @@ The source test/build Tasks do not receive Gitea, Harbor, or Cloudflare
|
|||||||
credentials. Credential Secrets are mounted only into the Tasks that require
|
credentials. Credential Secrets are mounted only into the Tasks that require
|
||||||
them.
|
them.
|
||||||
|
|
||||||
|
## External Setup
|
||||||
|
|
||||||
|
- Create the Infisical path and all six keys in the Secret Contract before the
|
||||||
|
first run.
|
||||||
|
- Point public DNS for `tekton-hook.dahyeon.us` to the same public ingress
|
||||||
|
endpoint as `tekton.dahyeon.us`.
|
||||||
|
- In Gitea, add an active push webhook for
|
||||||
|
`https://tekton-hook.dahyeon.us` with JSON content type and use the exact
|
||||||
|
`GITEA_WEBHOOK_SECRET` value as its signing secret.
|
||||||
|
- Protect `https://tekton.dahyeon.us` with an authentication layer such as
|
||||||
|
Cloudflare Access when build metadata and logs must not be public.
|
||||||
|
|
||||||
## Pipeline Contract
|
## Pipeline Contract
|
||||||
|
|
||||||
- Trigger: signed Gitea push event for exactly
|
- Trigger: signed Gitea push event for exactly
|
||||||
@@ -182,7 +268,7 @@ Never:
|
|||||||
## Success Criteria
|
## Success Criteria
|
||||||
|
|
||||||
- `TektonConfig` and all required controllers are Ready.
|
- `TektonConfig` and all required controllers are Ready.
|
||||||
- `https://tekton.dahyeon.us` returns HTTP 200 in read-only mode.
|
- `https://tekton.dahyeon.us` returns HTTP 200 with read-only mode disabled.
|
||||||
- A valid main push creates exactly one staging PipelineRun.
|
- A valid main push creates exactly one staging PipelineRun.
|
||||||
- Invalid signatures, repositories, branches, and malformed SHAs are rejected.
|
- Invalid signatures, repositories, branches, and malformed SHAs are rejected.
|
||||||
- Quality gates complete before side effects.
|
- Quality gates complete before side effects.
|
||||||
|
|||||||
@@ -23,3 +23,11 @@ spec:
|
|||||||
resourceGroupName: dahyeon
|
resourceGroupName: dahyeon
|
||||||
hostedZoneName: dahyeon.us
|
hostedZoneName: dahyeon.us
|
||||||
environment: AzurePublicCloud
|
environment: AzurePublicCloud
|
||||||
|
# rabbitmaus.com is outside the Azure DNS zone managed above.
|
||||||
|
# Public HTTP traffic for the ACME challenge must reach HAProxy.
|
||||||
|
- selector:
|
||||||
|
dnsNames:
|
||||||
|
- rabbitmaus.com
|
||||||
|
http01:
|
||||||
|
ingress:
|
||||||
|
ingressClassName: haproxy
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# Redis
|
||||||
|
|
||||||
|
`platform` 네임스페이스에서 heize에 단일 인스턴스로 배포합니다.
|
||||||
|
|
||||||
|
- 내부 주소: `redis.platform.svc.cluster.local:6379` (ClusterIP)
|
||||||
|
- 저장소: `local-path` PVC `10Gi`
|
||||||
|
- 메모리: `maxmemory 3GiB`, 컨테이너 요청 `4Gi` / 제한 `5Gi`
|
||||||
|
- 영속화: AOF `everysec`
|
||||||
|
- 메모리 정책: `noeviction` — 한도 도달 시 메모리를 추가로 사용하는 쓰기 실패
|
||||||
|
|
||||||
|
## 배포
|
||||||
|
|
||||||
|
1. [Infisical 설정](overlays/main/infisical-secret.yaml)의 `prod` 환경,
|
||||||
|
`/platform/redis` 경로에 `REDIS_USERNAME`, `REDIS_PASSWORD`를 등록합니다.
|
||||||
|
사용자명은 영문·숫자·`_`·`.`·`-`만 허용하며 `default`는 사용할 수 없습니다.
|
||||||
|
2. main 반영 후 `cluster-main` 동기화로 `platform` 네임스페이스가 생성됐는지 확인합니다.
|
||||||
|
3. Argo CD의 [redis Application](../../argocd/applications/redis.yaml)을 수동 동기화합니다.
|
||||||
|
4. `platform/redis-auth` Secret 생성과 `redis-0` Pod의 Ready 상태를 확인합니다.
|
||||||
|
|
||||||
|
매니페스트 확인:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl kustomize platform/redis/overlays/main
|
||||||
|
```
|
||||||
|
|
||||||
|
## 개발자 접속
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n platform port-forward svc/redis 16379:6379
|
||||||
|
```
|
||||||
|
|
||||||
|
명령을 실행한 상태에서 `127.0.0.1:16379`에 Redis 사용자명·비밀번호로 접속합니다.
|
||||||
|
|
||||||
|
## 운영
|
||||||
|
|
||||||
|
- heize 장애 시 Redis도 중단됩니다. 초기 운영에서는 이를 감수합니다.
|
||||||
|
- `10Gi`는 요청량이며 실제 디스크 사용 상한이 아닙니다. PVC 삭제 시 데이터가 삭제될 수 있습니다.
|
||||||
|
- AOF는 별도 백업을 대신하지 않습니다. 재작성 중 메모리·디스크 사용량은 배포 후 측정합니다.
|
||||||
|
- [알림 규칙](overlays/main/monitoring.yaml)은 포함돼 있으며, 실제 수신 채널은 별도 설정해야 합니다.
|
||||||
|
- 현재 인증 정보 변경은 Infisical 동기화 후 Pod 재시작이 필요합니다.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
kubectl -n platform rollout restart statefulset/redis
|
||||||
|
kubectl -n platform rollout status statefulset/redis
|
||||||
|
```
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- statefulset.yaml
|
||||||
|
- service.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: redis-config
|
||||||
|
files:
|
||||||
|
- redis.conf
|
||||||
|
- start-redis.sh
|
||||||
|
|
||||||
|
labels:
|
||||||
|
- includeSelectors: true
|
||||||
|
pairs:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
app.kubernetes.io/part-of: redis
|
||||||
|
app.kubernetes.io/managed-by: kustomize
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
bind 0.0.0.0
|
||||||
|
protected-mode yes
|
||||||
|
port 6379
|
||||||
|
dir /data
|
||||||
|
aclfile /run/redis-acl/users.acl
|
||||||
|
|
||||||
|
appendonly yes
|
||||||
|
appendfsync everysec
|
||||||
|
save ""
|
||||||
|
|
||||||
|
# Redis "gb" uses binary units: 3gb = 3 GiB.
|
||||||
|
# The container limit is 5Gi. This is a measured-risk starting point: heavy
|
||||||
|
# writes during AOF rewrite can exhaust the remaining fork/COW headroom.
|
||||||
|
# maxmemory is not a hard limit on total process memory.
|
||||||
|
maxmemory 3gb
|
||||||
|
# Preserve keys until the application explicitly deletes or expires them.
|
||||||
|
maxmemory-policy noeviction
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis-headless
|
||||||
|
spec:
|
||||||
|
clusterIP: None
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: redis
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis-metrics
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: metrics
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
port: 9121
|
||||||
|
targetPort: metrics
|
||||||
|
# Keep the exporter scrapeable while the Redis readiness probe fails.
|
||||||
|
publishNotReadyAddresses: true
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
port: 6379
|
||||||
|
targetPort: redis
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# Restrict usernames to a single ACL token; never interpolate raw passwords
|
||||||
|
# into Redis configuration syntax or command-line arguments.
|
||||||
|
case "${REDIS_USERNAME:-}" in
|
||||||
|
""|default|*[!a-zA-Z0-9_.-]*)
|
||||||
|
echo "REDIS_USERNAME must contain only letters, digits, _, . or - and must not be default." >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
if [ -z "${REDISCLI_AUTH:-}" ]; then
|
||||||
|
echo "REDIS_PASSWORD must not be empty." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
umask 077
|
||||||
|
password_hash="$(printf '%s' "$REDISCLI_AUTH" | sha256sum)"
|
||||||
|
password_hash="${password_hash%% *}"
|
||||||
|
printf 'user default off\nuser %s on #%s ~* &* +@all\n' \
|
||||||
|
"$REDIS_USERNAME" "$password_hash" > /run/redis-acl/users.acl
|
||||||
|
unset password_hash
|
||||||
|
|
||||||
|
exec redis-server /etc/redis/redis.conf
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
spec:
|
||||||
|
serviceName: redis-headless
|
||||||
|
replicas: 1
|
||||||
|
persistentVolumeClaimRetentionPolicy:
|
||||||
|
whenDeleted: Retain
|
||||||
|
whenScaled: Retain
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 999
|
||||||
|
runAsGroup: 1000
|
||||||
|
fsGroup: 1000
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: redis
|
||||||
|
image: redis:8.10.3-alpine
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- /etc/redis/start-redis.sh
|
||||||
|
env:
|
||||||
|
- name: REDIS_USERNAME
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: username
|
||||||
|
- name: REDISCLI_AUTH
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: password
|
||||||
|
ports:
|
||||||
|
- name: redis
|
||||||
|
containerPort: 6379
|
||||||
|
startupProbe:
|
||||||
|
exec:
|
||||||
|
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 120
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
livenessProbe:
|
||||||
|
exec:
|
||||||
|
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 4Gi
|
||||||
|
limits:
|
||||||
|
memory: 5Gi
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
capabilities:
|
||||||
|
drop: [ALL]
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/redis
|
||||||
|
readOnly: true
|
||||||
|
- name: acl
|
||||||
|
mountPath: /run/redis-acl
|
||||||
|
- name: exporter
|
||||||
|
image: oliver006/redis_exporter:v1.94.0
|
||||||
|
args:
|
||||||
|
- --redis.addr=redis://127.0.0.1:6379
|
||||||
|
- --redis-only-metrics
|
||||||
|
- --disable-scrape-endpoint
|
||||||
|
- --connection-timeout=5s
|
||||||
|
env:
|
||||||
|
- name: REDIS_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: username
|
||||||
|
- name: REDIS_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: redis-auth
|
||||||
|
key: password
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
containerPort: 9121
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 25m
|
||||||
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
memory: 128Mi
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop: [ALL]
|
||||||
|
volumes:
|
||||||
|
- name: acl
|
||||||
|
emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 1Mi
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: redis-config
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: data
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: local-path
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
apiVersion: secrets.infisical.com/v1beta1
|
||||||
|
kind: InfisicalStaticSecret
|
||||||
|
metadata:
|
||||||
|
name: redis-auth
|
||||||
|
spec:
|
||||||
|
infisicalAuthRef:
|
||||||
|
name: infisical-auth
|
||||||
|
namespace: infisical
|
||||||
|
syncOptions:
|
||||||
|
refreshInterval: 60s
|
||||||
|
instantUpdates: false
|
||||||
|
sources:
|
||||||
|
- projectId: 8520b8c0-3796-4fd6-a03c-6f0508c8c42c
|
||||||
|
environmentSlug: prod
|
||||||
|
secretPath: /platform/redis
|
||||||
|
recursive: false
|
||||||
|
targets:
|
||||||
|
- name: redis-auth
|
||||||
|
namespace: platform
|
||||||
|
kind: Secret
|
||||||
|
creationPolicy: Owner
|
||||||
|
secretType: Opaque
|
||||||
|
template:
|
||||||
|
engineVersion: v1
|
||||||
|
data:
|
||||||
|
username: "{{ .REDIS_USERNAME.Value }}"
|
||||||
|
password: "{{ .REDIS_PASSWORD.Value }}"
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
namespace: platform
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- ../../base
|
||||||
|
- infisical-secret.yaml
|
||||||
|
- monitoring.yaml
|
||||||
|
|
||||||
|
patches:
|
||||||
|
- path: statefulset-patch.yaml
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
labels:
|
||||||
|
release: monitoring
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: redis
|
||||||
|
app.kubernetes.io/component: metrics
|
||||||
|
endpoints:
|
||||||
|
- port: metrics
|
||||||
|
interval: 30s
|
||||||
|
scrapeTimeout: 10s
|
||||||
|
relabelings:
|
||||||
|
- targetLabel: job
|
||||||
|
replacement: redis
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
labels:
|
||||||
|
release: monitoring
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: redis
|
||||||
|
rules:
|
||||||
|
- alert: RedisUnavailable
|
||||||
|
expr: |
|
||||||
|
redis_up{namespace="platform",job="redis"} == 0
|
||||||
|
or up{namespace="platform",job="redis"} == 0
|
||||||
|
or absent(redis_up{namespace="platform",job="redis"})
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis or its metrics endpoint is unavailable
|
||||||
|
description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target.
|
||||||
|
- alert: RedisMetricsScrapeFailed
|
||||||
|
expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis metrics collection is failing
|
||||||
|
description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics.
|
||||||
|
- alert: RedisAofWriteFailed
|
||||||
|
expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis cannot persist writes to AOF
|
||||||
|
description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure.
|
||||||
|
- alert: RedisAofRewriteFailed
|
||||||
|
expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis AOF rewrite failed
|
||||||
|
description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds.
|
||||||
|
- alert: RedisWriteRejected
|
||||||
|
expr: |
|
||||||
|
increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0
|
||||||
|
or (
|
||||||
|
redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0
|
||||||
|
unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m
|
||||||
|
)
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis rejected writes due to memory or persistence errors
|
||||||
|
description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly.
|
||||||
|
- alert: RedisMemoryNearMaxmemory
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
redis_memory_used_bytes{namespace="platform",job="redis"}
|
||||||
|
- redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"}
|
||||||
|
) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis is approaching its maxmemory threshold
|
||||||
|
description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold.
|
||||||
|
- alert: RedisContainerMemoryHigh
|
||||||
|
expr: |
|
||||||
|
max by (namespace, pod, container) (
|
||||||
|
container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"}
|
||||||
|
)
|
||||||
|
/ max by (namespace, pod, container) (
|
||||||
|
kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"}
|
||||||
|
) > 0.85
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis container is approaching its memory limit
|
||||||
|
description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting.
|
||||||
|
- alert: RedisContainerOomKilled
|
||||||
|
expr: |
|
||||||
|
max by (namespace, pod, container) (
|
||||||
|
kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1
|
||||||
|
)
|
||||||
|
and on (namespace, pod, container)
|
||||||
|
(
|
||||||
|
max by (namespace, pod, container) (
|
||||||
|
increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m])
|
||||||
|
) > 0
|
||||||
|
)
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis was restarted after an OOM kill
|
||||||
|
description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks.
|
||||||
|
- alert: RedisAofFsyncDelayed
|
||||||
|
expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Redis AOF fsync is delayed
|
||||||
|
description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded.
|
||||||
|
- alert: RedisUnexpectedEviction
|
||||||
|
expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Redis evicted keys despite the intended noeviction policy
|
||||||
|
description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners.
|
||||||
|
- alert: RedisNodeDiskHeadroomLow
|
||||||
|
# local-path currently stores Redis on heize's root filesystem.
|
||||||
|
# If the storage path moves to a dedicated mount, change mountpoint.
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||||
|
* on (instance) group_left (nodename)
|
||||||
|
node_uname_info{job="node-exporter",nodename="heize"}
|
||||||
|
) < 10 * 1024 * 1024 * 1024
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: heize has less than 10GiB of free space for Redis AOF rewrites
|
||||||
|
description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: redis
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/environment: prod
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/hostname: heize
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
"""Validate rendered Redis resources and exercise alert behavior with promtool.
|
||||||
|
|
||||||
|
Requires kubectl and promtool on PATH (or --promtool /path/to/promtool).
|
||||||
|
Uses client dry-run only; never applies resources to the cluster.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
|
||||||
|
def parse_json_stream(text):
|
||||||
|
decoder = json.JSONDecoder()
|
||||||
|
documents = []
|
||||||
|
while text.strip():
|
||||||
|
text = text.lstrip()
|
||||||
|
document, end = decoder.raw_decode(text)
|
||||||
|
documents.append(document)
|
||||||
|
text = text[end:]
|
||||||
|
return documents
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--promtool", default="promtool")
|
||||||
|
args = parser.parse_args()
|
||||||
|
app = Path(__file__).resolve().parents[1]
|
||||||
|
result = subprocess.run(
|
||||||
|
["kubectl", "create", "--dry-run=client", "-k",
|
||||||
|
str(app / "overlays/main"), "-o", "json", "--request-timeout=10s"],
|
||||||
|
check=True, capture_output=True, text=True, encoding="utf-8",
|
||||||
|
)
|
||||||
|
resources = parse_json_stream(result.stdout)
|
||||||
|
sts = next(r for r in resources if r["kind"] == "StatefulSet")
|
||||||
|
containers = sts["spec"]["template"]["spec"]["containers"]
|
||||||
|
redis = next(c for c in containers if c["name"] == "redis")
|
||||||
|
assert redis["resources"]["limits"]["memory"] == "5Gi"
|
||||||
|
assert redis["resources"]["requests"]["memory"] == "4Gi"
|
||||||
|
assert all("cpu" not in c["resources"].get("limits", {}) for c in containers)
|
||||||
|
config = next(r for r in resources if r["kind"] == "ConfigMap")
|
||||||
|
assert "maxmemory 3gb\n" in config["data"]["redis.conf"]
|
||||||
|
assert "maxmemory-policy noeviction\n" in config["data"]["redis.conf"]
|
||||||
|
metrics_service = next(
|
||||||
|
r for r in resources if r["kind"] == "Service"
|
||||||
|
and r["metadata"]["name"] == "redis-metrics"
|
||||||
|
)
|
||||||
|
assert metrics_service["spec"]["publishNotReadyAddresses"] is True
|
||||||
|
monitor = next(r for r in resources if r["kind"] == "ServiceMonitor")
|
||||||
|
rule_resource = next(r for r in resources if r["kind"] == "PrometheusRule")
|
||||||
|
assert monitor["metadata"]["labels"]["release"] == "monitoring"
|
||||||
|
assert rule_resource["metadata"]["labels"]["release"] == "monitoring"
|
||||||
|
assert any(
|
||||||
|
relabel.get("targetLabel") == "job" and relabel.get("replacement") == "redis"
|
||||||
|
for relabel in monitor["spec"]["endpoints"][0]["relabelings"]
|
||||||
|
)
|
||||||
|
rules = {
|
||||||
|
rule["alert"]: rule
|
||||||
|
for group in rule_resource["spec"]["groups"]
|
||||||
|
for rule in group["rules"]
|
||||||
|
}
|
||||||
|
base = {"namespace": "platform", "job": "redis"}
|
||||||
|
container = {"namespace": "platform", "pod": "redis-0", "container": "redis"}
|
||||||
|
|
||||||
|
def series(metric, values, labels=None):
|
||||||
|
labels = base if labels is None else labels
|
||||||
|
selector = ",".join(f'{key}="{value}"' for key, value in labels.items())
|
||||||
|
return {"series": f"{metric}{{{selector}}}", "values": values}
|
||||||
|
|
||||||
|
tests = []
|
||||||
|
|
||||||
|
def case(name, alert, inputs, firing, labels=None, at="6m", healthy=True):
|
||||||
|
labels = base if labels is None else labels
|
||||||
|
expected = []
|
||||||
|
if firing:
|
||||||
|
expected = [{
|
||||||
|
"exp_labels": {**labels, **rules[alert]["labels"]},
|
||||||
|
"exp_annotations": rules[alert]["annotations"],
|
||||||
|
}]
|
||||||
|
tests.append({
|
||||||
|
"name": name, "interval": "30s",
|
||||||
|
"input_series": (
|
||||||
|
[series("redis_up", "1+0x30")] if healthy else []
|
||||||
|
) + inputs,
|
||||||
|
"alert_rule_test": [{
|
||||||
|
"eval_time": at, "alertname": alert, "exp_alerts": expected,
|
||||||
|
}],
|
||||||
|
})
|
||||||
|
|
||||||
|
case("healthy Redis", "RedisUnavailable", [], False)
|
||||||
|
case("missing exporter target", "RedisUnavailable", [], True, healthy=False)
|
||||||
|
case("Redis unreachable", "RedisUnavailable",
|
||||||
|
[series("redis_up", "0+0x30")], True, healthy=False)
|
||||||
|
case("HTTP endpoint down while Redis metric still exists", "RedisUnavailable",
|
||||||
|
[series("up", "0+0x30")], True)
|
||||||
|
case("brief outage is below hold duration", "RedisUnavailable",
|
||||||
|
[series("redis_up", "0+0x30")], False, at="1m", healthy=False)
|
||||||
|
case("exporter cannot read Redis INFO", "RedisMetricsScrapeFailed",
|
||||||
|
[series("redis_exporter_last_scrape_error", "1+0x30", {**base, "err": "NOPERM"})],
|
||||||
|
True, {**base, "err": "NOPERM"})
|
||||||
|
for metric, alert in [
|
||||||
|
("redis_aof_last_write_status", "RedisAofWriteFailed"),
|
||||||
|
("redis_aof_last_bgrewrite_status", "RedisAofRewriteFailed"),
|
||||||
|
]:
|
||||||
|
case(f"{metric} ok is one", alert, [series(metric, "1+0x30")], False)
|
||||||
|
case(f"{metric} failure is zero", alert, [series(metric, "0+0x30")], True)
|
||||||
|
for error in ["OOM", "MISCONF"]:
|
||||||
|
labels = {**base, "err": error}
|
||||||
|
case(f"new {error} counter starts at one", "RedisWriteRejected",
|
||||||
|
[series("redis_errors_total", "_ _ _ _ 1+0x26", labels)],
|
||||||
|
True, labels, at="3m")
|
||||||
|
case(f"old {error} error does not alert forever", "RedisWriteRejected",
|
||||||
|
[series("redis_errors_total", "1+0x30", labels)],
|
||||||
|
False, at="10m")
|
||||||
|
case("growing OOM error counter", "RedisWriteRejected",
|
||||||
|
[series("redis_errors_total", "0+1x30", {**base, "err": "OOM"})],
|
||||||
|
True, {**base, "err": "OOM"})
|
||||||
|
case("Redis memory high", "RedisMemoryNearMaxmemory", [
|
||||||
|
series("redis_memory_used_bytes", "3006477107+0x30"),
|
||||||
|
series("redis_mem_not_counted_for_eviction_bytes", "0+0x30"),
|
||||||
|
series("redis_memory_max_bytes", "3221225472+0x30"),
|
||||||
|
], True)
|
||||||
|
case("exclude buffers not counted by eviction", "RedisMemoryNearMaxmemory", [
|
||||||
|
series("redis_memory_used_bytes", "3221225472+0x30"),
|
||||||
|
series("redis_mem_not_counted_for_eviction_bytes", "1073741824+0x30"),
|
||||||
|
series("redis_memory_max_bytes", "3221225472+0x30"),
|
||||||
|
], False)
|
||||||
|
case("container limit evaluated independently from maxmemory", "RedisContainerMemoryHigh", [
|
||||||
|
series("container_memory_working_set_bytes", "4831838208+0x30", container),
|
||||||
|
series("kube_pod_container_resource_limits", "5368709120+0x30",
|
||||||
|
{**container, "resource": "memory"}),
|
||||||
|
], True, container)
|
||||||
|
for restarts, firing in [("0+1x30", True), ("1+0x30", False)]:
|
||||||
|
case(f"OOM reason with restarts {restarts}", "RedisContainerOomKilled", [
|
||||||
|
series("kube_pod_container_status_last_terminated_reason", "1+0x30",
|
||||||
|
{**container, "reason": "OOMKilled"}),
|
||||||
|
series("kube_pod_container_status_restarts_total", restarts, container),
|
||||||
|
], firing, container, at="12m")
|
||||||
|
for metric, alert in [
|
||||||
|
("redis_aof_delayed_fsync", "RedisAofFsyncDelayed"),
|
||||||
|
("redis_evicted_keys_total", "RedisUnexpectedEviction"),
|
||||||
|
]:
|
||||||
|
case(f"increasing {metric}", alert, [series(metric, "0+1x30")], True)
|
||||||
|
case(f"stable {metric}", alert, [series(metric, "1+0x30")], False)
|
||||||
|
disk = {"job": "node-exporter", "instance": "10.250.0.16:9100",
|
||||||
|
"mountpoint": "/", "fstype": "ext4", "device": "/dev/sda1"}
|
||||||
|
for node, free, firing in [
|
||||||
|
("heize", "9663676416", True),
|
||||||
|
("heize", "12884901888", False),
|
||||||
|
("suga", "9663676416", False),
|
||||||
|
]:
|
||||||
|
case(f"{node} free disk {free}", "RedisNodeDiskHeadroomLow", [
|
||||||
|
series("node_filesystem_avail_bytes", free + "+0x30", disk),
|
||||||
|
series("node_uname_info", "1+0x30",
|
||||||
|
{"job": "node-exporter", "instance": disk["instance"], "nodename": node}),
|
||||||
|
], firing, {**disk, "nodename": node})
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(prefix="redis-monitoring-") as directory:
|
||||||
|
rule_path = Path(directory) / "rules.json"
|
||||||
|
rule_path.write_text(json.dumps(rule_resource["spec"]), encoding="utf-8")
|
||||||
|
test_path = Path(directory) / "tests.json"
|
||||||
|
test_path.write_text(json.dumps({
|
||||||
|
"rule_files": [rule_path.as_posix()],
|
||||||
|
"evaluation_interval": "30s", "tests": tests,
|
||||||
|
}), encoding="utf-8")
|
||||||
|
subprocess.run([args.promtool, "check", "rules", str(rule_path)], check=True)
|
||||||
|
subprocess.run([args.promtool, "test", "rules", str(test_path)], check=True)
|
||||||
|
print(f"PASS: resource assertions and {len(tests)} alert scenarios")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# Preserve the existing names so Argo CD can adopt the shared policies in place.
|
||||||
|
# These policies apply to all services in tekton-ci, not only lolympus.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ResourceQuota
|
||||||
|
metadata:
|
||||||
|
name: lolympus-ci
|
||||||
|
namespace: tekton-ci
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tekton-ci
|
||||||
|
app.kubernetes.io/part-of: tekton-ci
|
||||||
|
app.kubernetes.io/managed-by: argocd
|
||||||
|
spec:
|
||||||
|
hard:
|
||||||
|
pods: "20"
|
||||||
|
# Reserve room for new runs between pruning the completed history to 10 runs.
|
||||||
|
persistentvolumeclaims: "15"
|
||||||
|
# Tekton steps run sequentially, but quota sums their container limits.
|
||||||
|
# Bound compute requests here; retain per-container limits in Tasks/LimitRange.
|
||||||
|
requests.cpu: "8"
|
||||||
|
requests.memory: 16Gi
|
||||||
|
requests.storage: 150Gi
|
||||||
|
count/pipelineruns.tekton.dev: "15"
|
||||||
|
count/taskruns.tekton.dev: "150"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: LimitRange
|
||||||
|
metadata:
|
||||||
|
name: lolympus-ci
|
||||||
|
namespace: tekton-ci
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tekton-ci
|
||||||
|
app.kubernetes.io/part-of: tekton-ci
|
||||||
|
app.kubernetes.io/managed-by: argocd
|
||||||
|
spec:
|
||||||
|
limits:
|
||||||
|
- type: Container
|
||||||
|
defaultRequest:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
default:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 4Gi
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: lolympus-tekton-eventlistener-interceptors
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- triggers.tekton.dev
|
||||||
|
resources:
|
||||||
|
- clusterinterceptors
|
||||||
|
- clustertriggerbindings
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: lolympus-tekton-eventlistener-interceptors
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: lolympus-tekton-eventlistener-interceptors
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: lolympus-event-listener
|
||||||
|
namespace: tekton-ci
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: url-shortener-tekton-eventlistener-interceptors
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- triggers.tekton.dev
|
||||||
|
resources:
|
||||||
|
- clusterinterceptors
|
||||||
|
- clustertriggerbindings
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: url-shortener-tekton-eventlistener-interceptors
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: url-shortener-tekton-eventlistener-interceptors
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: url-shortener-event-listener
|
||||||
|
namespace: tekton-ci
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: make-space-tekton-eventlistener-interceptors
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- triggers.tekton.dev
|
||||||
|
resources:
|
||||||
|
- clusterinterceptors
|
||||||
|
- clustertriggerbindings
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: make-space-tekton-eventlistener-interceptors
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: make-space-tekton-eventlistener-interceptors
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: make-space-event-listener
|
||||||
|
namespace: tekton-ci
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: blog-tekton-eventlistener-interceptors
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- triggers.tekton.dev
|
||||||
|
resources:
|
||||||
|
- clusterinterceptors
|
||||||
|
- clustertriggerbindings
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: blog-tekton-eventlistener-interceptors
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: blog-tekton-eventlistener-interceptors
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: blog-event-listener
|
||||||
|
namespace: tekton-ci
|
||||||
@@ -3,3 +3,5 @@ kind: Kustomization
|
|||||||
resources:
|
resources:
|
||||||
- tektonconfig.yaml
|
- tektonconfig.yaml
|
||||||
- ingress.yaml
|
- ingress.yaml
|
||||||
|
- eventlistener-rbac.yaml
|
||||||
|
- ci-resource-policy.yaml
|
||||||
@@ -6,12 +6,12 @@ spec:
|
|||||||
targetNamespace: tekton-pipelines
|
targetNamespace: tekton-pipelines
|
||||||
profile: all
|
profile: all
|
||||||
dashboard:
|
dashboard:
|
||||||
readonly: true
|
readonly: false
|
||||||
pruner:
|
pruner:
|
||||||
disabled: false
|
disabled: false
|
||||||
schedule: "0 */6 * * *"
|
schedule: "*/15 * * * *"
|
||||||
resources:
|
resources:
|
||||||
- taskrun
|
|
||||||
- pipelinerun
|
- pipelinerun
|
||||||
|
# Keep the newest completed runs across each namespace; child resources follow their owner.
|
||||||
keep: 10
|
keep: 10
|
||||||
prune-per-resource: true
|
# prune-per-resource defaults to false; the operator omits an explicit false.
|
||||||
Reference in new issue
Block a user