Compare commits

..
14 Commits
Author SHA1 Message Date
dahyeon d6cbf1f23f feat(blog): register branch-based GitOps delivery 2026-11-17 18:47:39 +09:00
dahyeon ecfc86c72f fix(tekton): avoid pruner default normalization drift 2026-11-09 18:41:09 +09:00
dahyeon 083eae7f8c refactor(tekton): own shared CI policies in platform 2026-11-09 18:35:24 +09:00
dahyeon 67356b1489 feat(redis): add shared Redis 8.10.3 to platform 2026-11-09 16:01:48 +09:00
dahyeon eb96f56349 feat(make-space): add namespaces, Tekton CI app, and prod auto-sync 2026-11-04 20:41:12 +09:00
dahyeon 2b65a583b9 feat(url-shortener): enable production GitOps auto-sync 2026-11-04 14:19:17 +09:00
dahyeon 1e20c435cd feat(tekton): reconcile url-shortener CI with scoped RBAC 2026-11-04 05:07:42 +09:00
dahyeon 6df004a1da chore(tekton): disable dashboard read-only mode 2026-09-27 01:36:41 +09:00
dahyeon 8b415c4e94 chore(tekton): retain five pipeline runs 2026-07-26 04:46:11 +09:00
dahyeon 2d0df191dc docs: add Tekton external setup steps 2026-07-18 08:26:11 +00:00
dahyeon b14c8e5b43 fix: grant scoped Tekton interceptor access 2026-07-18 08:03:02 +00:00
dahyeon 9ec1eebe2e feat: reconcile lolympus Tekton CI 2026-07-18 07:55:23 +00:00
dahyeon c2e98ca321 feat: add isolated Tekton CI namespace
Keep application PipelineRuns and their credentials separate from the operator-managed tekton-pipelines namespace.
2026-07-18 07:08:51 +00:00
dahyeon 79268872f6 docs: define Tekton staging delivery contract
Document repository ownership, credential boundaries, quality gates, immutable release identifiers, and the GitOps verification plan before adding CI resources.
2026-07-18 07:07:52 +00:00
25 changed files with 1287 additions and 4 deletions

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: blog-ci
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
project: ci
source:
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
targetRevision: main
path: ci/blog
destination:
server: https://kubernetes.default.svc
namespace: tekton-ci
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- SkipDryRunOnMissingResource=true
@@ -0,0 +1,22 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: lolympus-tournament-support-ci
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
project: ci
source:
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
targetRevision: main
path: ci/lolympus-tournament-support
destination:
server: https://kubernetes.default.svc
namespace: tekton-ci
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- SkipDryRunOnMissingResource=true
+22
View File
@@ -0,0 +1,22 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: make-space-ci
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
project: ci
source:
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
targetRevision: main
path: ci/make-space
destination:
server: https://kubernetes.default.svc
namespace: tekton-ci
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- SkipDryRunOnMissingResource=true
+16
View File
@@ -0,0 +1,16 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: redis
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "5"
spec:
project: platform
source:
repoURL: https://git.dahyeon.us/dahyeon/platform-state.git
targetRevision: main
path: platform/redis/overlays/main
destination:
server: https://kubernetes.default.svc
namespace: platform
+22
View File
@@ -0,0 +1,22 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: url-shortener-ci
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
project: ci
source:
repoURL: https://git.dahyeon.us/dahyeon/service-state.git
targetRevision: main
path: ci/url-shortener
destination:
server: https://kubernetes.default.svc
namespace: tekton-ci
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- SkipDryRunOnMissingResource=true
+10
View File
@@ -26,3 +26,13 @@ spec:
destination: destination:
server: https://kubernetes.default.svc server: https://kubernetes.default.svc
namespace: "{{index .path.segments 1}}-prod" namespace: "{{index .path.segments 1}}-prod"
templatePatch: |
{{- if or (eq (index .path.segments 1) "url-shortener") (eq (index .path.segments 1) "make-space") (eq (index .path.segments 1) "blog") }}
spec:
syncPolicy:
automated:
prune: true
selfHeal: true
{{- else }}
{}
{{- end }}
+37
View File
@@ -0,0 +1,37 @@
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: ci
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "-20"
spec:
description: Namespaced CI delivery resources from service-state
sourceRepos:
- https://git.dahyeon.us/dahyeon/service-state.git
destinations:
- server: https://kubernetes.default.svc
namespace: tekton-ci
namespaceResourceWhitelist:
- group: ""
kind: ServiceAccount
- group: rbac.authorization.k8s.io
kind: Role
- group: rbac.authorization.k8s.io
kind: RoleBinding
- group: networking.k8s.io
kind: Ingress
- group: secrets.infisical.com
kind: InfisicalStaticSecret
- group: tekton.dev
kind: Task
- group: tekton.dev
kind: Pipeline
- group: triggers.tekton.dev
kind: EventListener
- group: triggers.tekton.dev
kind: TriggerBinding
- group: triggers.tekton.dev
kind: TriggerTemplate
orphanedResources:
warn: true
+30
View File
@@ -35,6 +35,11 @@ metadata:
--- ---
apiVersion: v1 apiVersion: v1
kind: Namespace kind: Namespace
metadata:
name: tekton-ci
---
apiVersion: v1
kind: Namespace
metadata: metadata:
name: api-staging name: api-staging
--- ---
@@ -82,3 +87,28 @@ apiVersion: v1
kind: Namespace kind: Namespace
metadata: metadata:
name: persona-lenz-staging name: persona-lenz-staging
---
apiVersion: v1
kind: Namespace
metadata:
name: make-space-staging
---
apiVersion: v1
kind: Namespace
metadata:
name: make-space-prod
---
apiVersion: v1
kind: Namespace
metadata:
name: blog-staging
---
apiVersion: v1
kind: Namespace
metadata:
name: blog-prod
---
apiVersion: v1
kind: Namespace
metadata:
name: platform
+289
View File
@@ -0,0 +1,289 @@
# Spec: Tekton CI/CD for lolympus-tournament-support
This document's original main-to-staging/manual-production contract is specific
to lolympus. Blog follows make-space: source `staging` and `prod` branches each
run a dedicated pipeline and both environments auto-sync. Blog registration is
`argocd/applications/blog-ci.yaml`; its complete delivery/secret contract is in
service-state `ci/blog/README.md` and `apps/blog/README.md`.
## Objective
Build a GitOps-native staging delivery path for the lolympus monorepo on the
existing k3s cluster.
A push to the source repository's `main` branch must:
1. validate the API and web applications;
2. build and push an immutable API image to Harbor;
3. deploy the web application to the Cloudflare Pages `staging` branch; and
4. update only the staging image tag in `service-state`.
Argo CD remains the only component that deploys Kubernetes workloads.
Production promotion remains a separate PR/manual operation.
## Tech Stack
- Kubernetes: k3s v1.35.5+k3s1, amd64, containerd
- GitOps: Argo CD with Kustomize
- CI: Tekton Operator v0.79.1, Pipelines v1.9.3, Triggers v0.35.0
- Image builder: BuildKit v0.31.1 rootless, pinned by digest
- Source and GitOps hosting: Gitea at `git.dahyeon.us`
- Image registry: `registry.dahyeon.us/lolympus/lolympus-tournament-support`
- Web hosting: Cloudflare Pages project `lolympus`
- Secret management: Infisical Secrets Operator
## Commands
Render platform state:
```bash
kubectl kustomize clusters/main
kubectl kustomize platform/tekton/config
```
Render and validate CI state:
```bash
kubectl kustomize ci/lolympus-tournament-support
kubectl apply --dry-run=server -k ci/lolympus-tournament-support
```
Application quality gates:
```bash
corepack pnpm install --frozen-lockfile
corepack pnpm --filter api exec eslint "{src,apps,libs,test}/**/*.ts" --max-warnings=0
corepack pnpm --filter web lint
corepack pnpm --filter api test -- --runInBand
corepack pnpm --filter web test
node --test scripts/deploy-cloudflare-pages.test.mjs
corepack pnpm --filter api build
corepack pnpm --filter web build:staging
corepack pnpm audit --prod --audit-level high
```
Runtime verification:
```bash
kubectl get tektonconfig,tektondashboard
kubectl get pipelinerun,taskrun -n tekton-ci
kubectl logs -n tekton-ci -l tekton.dev/pipelineRun=<run-name> --all-containers
```
## Project Structure
```text
platform-state/
docs/tekton-ci-cd.md
clusters/main/namespaces.yaml
argocd/projects/ci.yaml
argocd/applications/lolympus-tournament-support-ci.yaml
service-state/
ci/lolympus-tournament-support/
kustomization.yaml
foundation.yaml
infisical-secret.yaml
tasks.yaml
pipeline.yaml
trigger.yaml
ingress.yaml
```
Tekton controllers stay in `tekton-pipelines`. Application PipelineRuns and
their credentials run in the isolated `tekton-ci` namespace.
The `tekton-ci` namespace is shared by all service CI applications. Its
ResourceQuota and LimitRange are owned by the `tekton-config` Argo CD Application
from `platform/tekton/config/ci-resource-policy.yaml`. Their legacy resource name
`lolympus-ci` is retained to transfer ownership without recreating either object;
it does not restrict the policies to the lolympus service. The service `ci`
AppProject does not permit ResourceQuota or LimitRange resources.
The shared quota permits at most 15 PipelineRuns, 15 PVCs requesting a total of
150 GiB, and 150 TaskRuns. The job-based pruner runs at minute 00, 15, 30, and 45
of every hour and retains the newest 10 completed PipelineRuns per namespace,
across pipeline names and success/failure outcomes. Running and pending runs are
protected and are additional to the retained completed history. Child TaskRuns,
Pods, and workspace PVCs follow their owning PipelineRun's deletion; TaskRuns
are not pruned independently. If 15 runs accumulate before pruning frees a slot,
the quota rejects further PipelineRun creation.
### Transferring the shared resource policies
This transfer spans two repositories with independent automatic synchronization.
Do not rely on Application sync waves to order commits across the repositories.
For an existing cluster where `lolympus-tournament-support-ci` owns the policies:
1. Wait for active sync operations to finish. Temporarily disable automated sync
on `platform-root`, `tekton-config`, and `lolympus-tournament-support-ci`,
preserving their existing sync policy for restoration.
2. Record the UID and tracking annotation of both existing `lolympus-ci` objects
in `tekton-ci`. Add the temporary resource annotation
`argocd.argoproj.io/sync-options: Prune=false` to both objects.
3. Publish the service-state removal and platform-state adoption commits. Sync
`lolympus-tournament-support-ci` to the removal commit first. The protected
policies must remain present even though they are no longer desired there.
4. Sync `tekton-config` to the adoption commit. Verify both objects retain their
UIDs and now have tracking IDs beginning with `tekton-config:`. Verify the
quota values and the generated pruner CronJob's schedule and keep arguments.
5. Refresh the old Application and confirm it no longer tracks either policy.
Remove only the temporary `Prune=false` annotation after adoption succeeds.
6. Restore automated sync on the child Applications, then sync `platform-root`
to apply the tightened `ci` AppProject permissions and restore its automation.
Confirm all three Applications are Synced and Healthy.
If adoption fails, leave the existing policies protected and avoid syncing the
new AppProject restrictions until ownership is resolved. Do not delete and
recreate the policies or remove unrelated sync-option annotations.
## Registering Additional Service CI
CI directories in `service-state` are registered explicitly; the service
ApplicationSets discover only `apps/*/overlays/*`.
For URL shortener, `argocd/applications/url-shortener-ci.yaml` reconciles
`service-state/ci/url-shortener` into `tekton-ci` using the `ci` project.
The platform root Application discovers this registration recursively.
Cluster-scoped EventListener RBAC belongs in
`platform/tekton/config/eventlistener-rbac.yaml`. Each listener receives a
dedicated ClusterRole and ClusterRoleBinding allowing only `get`, `list`, and
`watch` on `clusterinterceptors` and `clustertriggerbindings`.
Keep ServiceAccounts, Roles, and RoleBindings in `service-state`; remove the
URL shortener ClusterRoleBinding from its `foundation.yaml` when adopting this
registration. The `ci` project intentionally does not allow cluster resources.
Before the first URL shortener run, configure its Infisical secrets and Gitea
push webhook at `https://tekton-hook.dahyeon.us/url-shortener` as defined in
`service-state/ci/url-shortener`.
## Code Style
Use stable Tekton APIs, explicit names, immutable image references, bounded
resources, and namespace-scoped RBAC.
```yaml
apiVersion: tekton.dev/v1
kind: Task
metadata:
name: lolympus-quality
spec:
workspaces:
- name: source
steps:
- name: test-api
image: node:26-bookworm-slim@sha256:<pinned-digest>
workingDir: $(workspaces.source.path)
script: |
#!/bin/sh
set -eu
pnpm --filter api test --runInBand
```
Do not use mutable image tags such as `latest`, inline credentials, or direct
`kubectl set image` deployment mutations.
The pinned Node 26 image does not include Corepack. The Tasks bootstrap exactly
`pnpm@10.4.1` into the shared workspace before running package scripts.
## Secret Contract
Infisical project `8520b8c0-3796-4fd6-a03c-6f0508c8c42c`, environment
`prod`, path `/platform/tekton/lolympus-tournament-support` must contain:
- `GITEA_USERNAME`: bot account with source read and service-state write access
- `GITEA_TOKEN`: token for that bot account
- `HARBOR_DOCKER_CONFIG_JSON`: Docker config JSON for a project-scoped robot
- `CLOUDFLARE_API_TOKEN`: token scoped to Pages deployments
- `CLOUDFLARE_ACCOUNT_ID`: Cloudflare account identifier
- `GITEA_WEBHOOK_SECRET`: high-entropy webhook signing secret
The source test/build Tasks do not receive Gitea, Harbor, or Cloudflare
credentials. Credential Secrets are mounted only into the Tasks that require
them.
## External Setup
- Create the Infisical path and all six keys in the Secret Contract before the
first run.
- Point public DNS for `tekton-hook.dahyeon.us` to the same public ingress
endpoint as `tekton.dahyeon.us`.
- In Gitea, add an active push webhook for
`https://tekton-hook.dahyeon.us` with JSON content type and use the exact
`GITEA_WEBHOOK_SECRET` value as its signing secret.
- Protect `https://tekton.dahyeon.us` with an authentication layer such as
Cloudflare Access when build metadata and logs must not be public.
## Pipeline Contract
- Trigger: signed Gitea push event for exactly
`dahyeon/lolympus-tournament-support` and `refs/heads/main`
- Source revision: the full 40-character commit SHA from the webhook
- Image tag: `sha-<full-commit-sha>`
- Cloudflare Pages branch: `staging`
- GitOps target:
`apps/lolympus-tournament-support/overlays/staging/kustomization.yaml`
- Superseded runs must not update `service-state`
- A failure in any quality, image, or Cloudflare step must prevent GitOps update
- Pipeline timeout: 45 minutes
- Workspace: 10 GiB `local-path` volume claim per PipelineRun
## Testing Strategy
- Static: `git diff --check` and Kustomize rendering for every state repo
- Schema: client dry-run before namespace creation, server dry-run afterwards
- Application: run every command listed under Application quality gates
- Integration: send a signed test webhook and verify one PipelineRun is created
- End-to-end: verify Harbor tag, Cloudflare staging deployment, service-state
commit, and Argo CD staging sync all refer to the same source SHA
- Rollback: revert the service-state tag commit and manually redeploy the prior
Cloudflare source revision if required
## Boundaries
Always:
- use Argo CD for persistent cluster resources;
- use immutable source and image revisions;
- validate webhook signatures and repository/branch fields;
- keep RBAC namespace-scoped and resource-limited;
- keep production promotion manual.
Ask first:
- enabling automatic production sync;
- changing the Harbor repository or Cloudflare project;
- granting CI credentials access to additional repositories;
- enabling privileged image builds.
Never:
- store secret values in Git;
- pass a repository URL from an untrusted webhook directly to a clone Task;
- expose credentials to application test or build steps;
- mutate staging Deployments directly with kubectl;
- deploy an unverified pull request with production-capable credentials.
## Success Criteria
- `TektonConfig` and all required controllers are Ready.
- `https://tekton.dahyeon.us` returns HTTP 200 with read-only mode disabled.
- A valid main push creates exactly one staging PipelineRun.
- Invalid signatures, repositories, branches, and malformed SHAs are rejected.
- Quality gates complete before side effects.
- The Harbor tag, Cloudflare staging deployment, and service-state staging tag
identify the same source commit.
- Argo CD automatically deploys staging and leaves production unchanged.
- Finished runs and their PVCs are pruned according to the configured policy.
## Implementation Plan
1. Add `tekton-ci` namespace and bounded namespace resources.
2. Add Infisical-backed, purpose-specific CI Secrets.
3. Add clone, quality, BuildKit, latest-revision, Cloudflare, and GitOps Tasks.
4. Compose the staging Pipeline and signed Gitea Trigger.
5. Add a TLS Ingress at `tekton-hook.dahyeon.us`.
6. Add the restricted Argo CD project and Application.
7. Validate a signed webhook end to end after the Infisical values exist.
@@ -23,3 +23,11 @@ spec:
resourceGroupName: dahyeon resourceGroupName: dahyeon
hostedZoneName: dahyeon.us hostedZoneName: dahyeon.us
environment: AzurePublicCloud environment: AzurePublicCloud
# rabbitmaus.com is outside the Azure DNS zone managed above.
# Public HTTP traffic for the ACME challenge must reach HAProxy.
- selector:
dnsNames:
- rabbitmaus.com
http01:
ingress:
ingressClassName: haproxy
+45
View File
@@ -0,0 +1,45 @@
# Redis
`platform` 네임스페이스에서 heize에 단일 인스턴스로 배포합니다.
- 내부 주소: `redis.platform.svc.cluster.local:6379` (ClusterIP)
- 저장소: `local-path` PVC `10Gi`
- 메모리: `maxmemory 3GiB`, 컨테이너 요청 `4Gi` / 제한 `5Gi`
- 영속화: AOF `everysec`
- 메모리 정책: `noeviction` — 한도 도달 시 메모리를 추가로 사용하는 쓰기 실패
## 배포
1. [Infisical 설정](overlays/main/infisical-secret.yaml)의 `prod` 환경,
`/platform/redis` 경로에 `REDIS_USERNAME`, `REDIS_PASSWORD`를 등록합니다.
사용자명은 영문·숫자·`_`·`.`·`-`만 허용하며 `default`는 사용할 수 없습니다.
2. main 반영 후 `cluster-main` 동기화로 `platform` 네임스페이스가 생성됐는지 확인합니다.
3. Argo CD의 [redis Application](../../argocd/applications/redis.yaml)을 수동 동기화합니다.
4. `platform/redis-auth` Secret 생성과 `redis-0` Pod의 Ready 상태를 확인합니다.
매니페스트 확인:
```sh
kubectl kustomize platform/redis/overlays/main
```
## 개발자 접속
```sh
kubectl -n platform port-forward svc/redis 16379:6379
```
명령을 실행한 상태에서 `127.0.0.1:16379`에 Redis 사용자명·비밀번호로 접속합니다.
## 운영
- heize 장애 시 Redis도 중단됩니다. 초기 운영에서는 이를 감수합니다.
- `10Gi`는 요청량이며 실제 디스크 사용 상한이 아닙니다. PVC 삭제 시 데이터가 삭제될 수 있습니다.
- AOF는 별도 백업을 대신하지 않습니다. 재작성 중 메모리·디스크 사용량은 배포 후 측정합니다.
- [알림 규칙](overlays/main/monitoring.yaml)은 포함돼 있으며, 실제 수신 채널은 별도 설정해야 합니다.
- 현재 인증 정보 변경은 Infisical 동기화 후 Pod 재시작이 필요합니다.
```sh
kubectl -n platform rollout restart statefulset/redis
kubectl -n platform rollout status statefulset/redis
```
+19
View File
@@ -0,0 +1,19 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- statefulset.yaml
- service.yaml
configMapGenerator:
- name: redis-config
files:
- redis.conf
- start-redis.sh
labels:
- includeSelectors: true
pairs:
app.kubernetes.io/name: redis
app.kubernetes.io/part-of: redis
app.kubernetes.io/managed-by: kustomize
+17
View File
@@ -0,0 +1,17 @@
bind 0.0.0.0
protected-mode yes
port 6379
dir /data
aclfile /run/redis-acl/users.acl
appendonly yes
appendfsync everysec
save ""
# Redis "gb" uses binary units: 3gb = 3 GiB.
# The container limit is 5Gi. This is a measured-risk starting point: heavy
# writes during AOF rewrite can exhaust the remaining fork/COW headroom.
# maxmemory is not a hard limit on total process memory.
maxmemory 3gb
# Preserve keys until the application explicitly deletes or expires them.
maxmemory-policy noeviction
+42
View File
@@ -0,0 +1,42 @@
apiVersion: v1
kind: Service
metadata:
name: redis-headless
spec:
clusterIP: None
selector:
app.kubernetes.io/name: redis
ports:
- name: redis
port: 6379
targetPort: redis
---
apiVersion: v1
kind: Service
metadata:
name: redis-metrics
labels:
app.kubernetes.io/component: metrics
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: redis
ports:
- name: metrics
port: 9121
targetPort: metrics
# Keep the exporter scrapeable while the Redis readiness probe fails.
publishNotReadyAddresses: true
---
apiVersion: v1
kind: Service
metadata:
name: redis
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: redis
ports:
- name: redis
port: 6379
targetPort: redis
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
# Restrict usernames to a single ACL token; never interpolate raw passwords
# into Redis configuration syntax or command-line arguments.
case "${REDIS_USERNAME:-}" in
""|default|*[!a-zA-Z0-9_.-]*)
echo "REDIS_USERNAME must contain only letters, digits, _, . or - and must not be default." >&2
exit 1
;;
esac
if [ -z "${REDISCLI_AUTH:-}" ]; then
echo "REDIS_PASSWORD must not be empty." >&2
exit 1
fi
umask 077
password_hash="$(printf '%s' "$REDISCLI_AUTH" | sha256sum)"
password_hash="${password_hash%% *}"
printf 'user default off\nuser %s on #%s ~* &* +@all\n' \
"$REDIS_USERNAME" "$password_hash" > /run/redis-acl/users.acl
unset password_hash
exec redis-server /etc/redis/redis.conf
+132
View File
@@ -0,0 +1,132 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis
spec:
serviceName: redis-headless
replicas: 1
persistentVolumeClaimRetentionPolicy:
whenDeleted: Retain
whenScaled: Retain
selector:
matchLabels:
app.kubernetes.io/name: redis
template:
metadata:
labels:
app.kubernetes.io/name: redis
spec:
automountServiceAccountToken: false
terminationGracePeriodSeconds: 60
securityContext:
runAsNonRoot: true
runAsUser: 999
runAsGroup: 1000
fsGroup: 1000
seccompProfile:
type: RuntimeDefault
containers:
- name: redis
image: redis:8.10.3-alpine
imagePullPolicy: IfNotPresent
command:
- sh
- /etc/redis/start-redis.sh
env:
- name: REDIS_USERNAME
valueFrom:
secretKeyRef:
name: redis-auth
key: username
- name: REDISCLI_AUTH
valueFrom:
secretKeyRef:
name: redis-auth
key: password
ports:
- name: redis
containerPort: 6379
startupProbe:
exec:
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 120
readinessProbe:
exec:
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
periodSeconds: 5
timeoutSeconds: 3
livenessProbe:
exec:
command: [sh, -ec, 'test "$(redis-cli --user "$REDIS_USERNAME" ping)" = PONG']
periodSeconds: 10
timeoutSeconds: 3
resources:
requests:
cpu: 100m
memory: 4Gi
limits:
memory: 5Gi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
volumeMounts:
- name: data
mountPath: /data
- name: config
mountPath: /etc/redis
readOnly: true
- name: acl
mountPath: /run/redis-acl
- name: exporter
image: oliver006/redis_exporter:v1.94.0
args:
- --redis.addr=redis://127.0.0.1:6379
- --redis-only-metrics
- --disable-scrape-endpoint
- --connection-timeout=5s
env:
- name: REDIS_USER
valueFrom:
secretKeyRef:
name: redis-auth
key: username
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-auth
key: password
ports:
- name: metrics
containerPort: 9121
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: [ALL]
volumes:
- name: acl
emptyDir:
medium: Memory
sizeLimit: 1Mi
- name: config
configMap:
name: redis-config
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: local-path
resources:
requests:
storage: 10Gi
@@ -0,0 +1,27 @@
apiVersion: secrets.infisical.com/v1beta1
kind: InfisicalStaticSecret
metadata:
name: redis-auth
spec:
infisicalAuthRef:
name: infisical-auth
namespace: infisical
syncOptions:
refreshInterval: 60s
instantUpdates: false
sources:
- projectId: 8520b8c0-3796-4fd6-a03c-6f0508c8c42c
environmentSlug: prod
secretPath: /platform/redis
recursive: false
targets:
- name: redis-auth
namespace: platform
kind: Secret
creationPolicy: Owner
secretType: Opaque
template:
engineVersion: v1
data:
username: "{{ .REDIS_USERNAME.Value }}"
password: "{{ .REDIS_PASSWORD.Value }}"
@@ -0,0 +1,12 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: platform
resources:
- ../../base
- infisical-secret.yaml
- monitoring.yaml
patches:
- path: statefulset-patch.yaml
@@ -0,0 +1,147 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: redis
labels:
release: monitoring
spec:
selector:
matchLabels:
app.kubernetes.io/name: redis
app.kubernetes.io/component: metrics
endpoints:
- port: metrics
interval: 30s
scrapeTimeout: 10s
relabelings:
- targetLabel: job
replacement: redis
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: redis
labels:
release: monitoring
spec:
groups:
- name: redis
rules:
- alert: RedisUnavailable
expr: |
redis_up{namespace="platform",job="redis"} == 0
or up{namespace="platform",job="redis"} == 0
or absent(redis_up{namespace="platform",job="redis"})
for: 2m
labels:
severity: critical
annotations:
summary: Redis or its metrics endpoint is unavailable
description: Check redis-0, the exporter, Infisical credentials, and the ServiceMonitor target.
- alert: RedisMetricsScrapeFailed
expr: redis_exporter_last_scrape_error{namespace="platform",job="redis"} != 0
for: 2m
labels:
severity: warning
annotations:
summary: Redis metrics collection is failing
description: Inspect exporter logs and ACL permissions; a healthy HTTP endpoint alone does not guarantee complete Redis metrics.
- alert: RedisAofWriteFailed
expr: redis_aof_last_write_status{namespace="platform",job="redis"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: Redis cannot persist writes to AOF
description: Check heize disk space, filesystem errors and Redis logs. PING may still succeed; restarting alone will not fix storage failure.
- alert: RedisAofRewriteFailed
expr: redis_aof_last_bgrewrite_status{namespace="platform",job="redis"} == 0
for: 5m
labels:
severity: warning
annotations:
summary: Redis AOF rewrite failed
description: Check fork/COW memory and free disk space. The AOF may continue growing until a rewrite succeeds.
- alert: RedisWriteRejected
expr: |
increase(redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"}[5m]) > 0
or (
redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} > 0
unless redis_errors_total{namespace="platform",job="redis",err=~"OOM|MISCONF"} offset 5m
)
labels:
severity: critical
annotations:
summary: Redis rejected writes due to memory or persistence errors
description: Cache writes and lock acquisition may fail. Inspect the OOM or MISCONF error and handle failed lock acquisition explicitly.
- alert: RedisMemoryNearMaxmemory
expr: |
(
redis_memory_used_bytes{namespace="platform",job="redis"}
- redis_mem_not_counted_for_eviction_bytes{namespace="platform",job="redis"}
) / redis_memory_max_bytes{namespace="platform",job="redis"} > 0.85
for: 5m
labels:
severity: warning
annotations:
summary: Redis is approaching its maxmemory threshold
description: More than 85% of maxmemory is in use. Under noeviction, new memory-allocating writes will fail at the threshold.
- alert: RedisContainerMemoryHigh
expr: |
max by (namespace, pod, container) (
container_memory_working_set_bytes{namespace="platform",pod="redis-0",container="redis"}
)
/ max by (namespace, pod, container) (
kube_pod_container_resource_limits{namespace="platform",pod="redis-0",container="redis",resource="memory"}
) > 0.85
for: 1m
labels:
severity: warning
annotations:
summary: Redis container is approaching its memory limit
description: Check AOF copy-on-write and RSS. The 5Gi container limit is independent of the 3GiB maxmemory setting.
- alert: RedisContainerOomKilled
expr: |
max by (namespace, pod, container) (
kube_pod_container_status_last_terminated_reason{namespace="platform",pod="redis-0",container="redis",reason="OOMKilled"} == 1
)
and on (namespace, pod, container)
(
max by (namespace, pod, container) (
increase(kube_pod_container_status_restarts_total{namespace="platform",pod="redis-0",container="redis"}[10m])
) > 0
)
labels:
severity: critical
annotations:
summary: Redis was restarted after an OOM kill
description: Review maxmemory and container headroom against rewrite load. A restart can also lose recently acquired locks.
- alert: RedisAofFsyncDelayed
expr: increase(redis_aof_delayed_fsync{namespace="platform",job="redis"}[5m]) > 0
labels:
severity: warning
annotations:
summary: Redis AOF fsync is delayed
description: Inspect disk latency and competing heize workloads; the expected everysec durability window may be exceeded.
- alert: RedisUnexpectedEviction
expr: increase(redis_evicted_keys_total{namespace="platform",job="redis"}[5m]) > 0
labels:
severity: critical
annotations:
summary: Redis evicted keys despite the intended noeviction policy
description: Check runtime configuration drift. Evicting a live lock can permit concurrent owners.
- alert: RedisNodeDiskHeadroomLow
# local-path currently stores Redis on heize's root filesystem.
# If the storage path moves to a dedicated mount, change mountpoint.
expr: |
(
node_filesystem_avail_bytes{job="node-exporter",mountpoint="/",fstype!~"tmpfs|overlay"}
* on (instance) group_left (nodename)
node_uname_info{job="node-exporter",nodename="heize"}
) < 10 * 1024 * 1024 * 1024
for: 5m
labels:
severity: warning
annotations:
summary: heize has less than 10GiB of free space for Redis AOF rewrites
description: Free or expand storage before AOF growth and rewrite exhaust the shared root filesystem. The 10Gi PVC request does not reserve disk space.
@@ -0,0 +1,12 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis
spec:
template:
metadata:
labels:
app.kubernetes.io/environment: prod
spec:
nodeSelector:
kubernetes.io/hostname: heize
+173
View File
@@ -0,0 +1,173 @@
"""Validate rendered Redis resources and exercise alert behavior with promtool.
Requires kubectl and promtool on PATH (or --promtool /path/to/promtool).
Uses client dry-run only; never applies resources to the cluster.
"""
import argparse
import json
from pathlib import Path
import subprocess
import tempfile
def parse_json_stream(text):
decoder = json.JSONDecoder()
documents = []
while text.strip():
text = text.lstrip()
document, end = decoder.raw_decode(text)
documents.append(document)
text = text[end:]
return documents
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--promtool", default="promtool")
args = parser.parse_args()
app = Path(__file__).resolve().parents[1]
result = subprocess.run(
["kubectl", "create", "--dry-run=client", "-k",
str(app / "overlays/main"), "-o", "json", "--request-timeout=10s"],
check=True, capture_output=True, text=True, encoding="utf-8",
)
resources = parse_json_stream(result.stdout)
sts = next(r for r in resources if r["kind"] == "StatefulSet")
containers = sts["spec"]["template"]["spec"]["containers"]
redis = next(c for c in containers if c["name"] == "redis")
assert redis["resources"]["limits"]["memory"] == "5Gi"
assert redis["resources"]["requests"]["memory"] == "4Gi"
assert all("cpu" not in c["resources"].get("limits", {}) for c in containers)
config = next(r for r in resources if r["kind"] == "ConfigMap")
assert "maxmemory 3gb\n" in config["data"]["redis.conf"]
assert "maxmemory-policy noeviction\n" in config["data"]["redis.conf"]
metrics_service = next(
r for r in resources if r["kind"] == "Service"
and r["metadata"]["name"] == "redis-metrics"
)
assert metrics_service["spec"]["publishNotReadyAddresses"] is True
monitor = next(r for r in resources if r["kind"] == "ServiceMonitor")
rule_resource = next(r for r in resources if r["kind"] == "PrometheusRule")
assert monitor["metadata"]["labels"]["release"] == "monitoring"
assert rule_resource["metadata"]["labels"]["release"] == "monitoring"
assert any(
relabel.get("targetLabel") == "job" and relabel.get("replacement") == "redis"
for relabel in monitor["spec"]["endpoints"][0]["relabelings"]
)
rules = {
rule["alert"]: rule
for group in rule_resource["spec"]["groups"]
for rule in group["rules"]
}
base = {"namespace": "platform", "job": "redis"}
container = {"namespace": "platform", "pod": "redis-0", "container": "redis"}
def series(metric, values, labels=None):
labels = base if labels is None else labels
selector = ",".join(f'{key}="{value}"' for key, value in labels.items())
return {"series": f"{metric}{{{selector}}}", "values": values}
tests = []
def case(name, alert, inputs, firing, labels=None, at="6m", healthy=True):
labels = base if labels is None else labels
expected = []
if firing:
expected = [{
"exp_labels": {**labels, **rules[alert]["labels"]},
"exp_annotations": rules[alert]["annotations"],
}]
tests.append({
"name": name, "interval": "30s",
"input_series": (
[series("redis_up", "1+0x30")] if healthy else []
) + inputs,
"alert_rule_test": [{
"eval_time": at, "alertname": alert, "exp_alerts": expected,
}],
})
case("healthy Redis", "RedisUnavailable", [], False)
case("missing exporter target", "RedisUnavailable", [], True, healthy=False)
case("Redis unreachable", "RedisUnavailable",
[series("redis_up", "0+0x30")], True, healthy=False)
case("HTTP endpoint down while Redis metric still exists", "RedisUnavailable",
[series("up", "0+0x30")], True)
case("brief outage is below hold duration", "RedisUnavailable",
[series("redis_up", "0+0x30")], False, at="1m", healthy=False)
case("exporter cannot read Redis INFO", "RedisMetricsScrapeFailed",
[series("redis_exporter_last_scrape_error", "1+0x30", {**base, "err": "NOPERM"})],
True, {**base, "err": "NOPERM"})
for metric, alert in [
("redis_aof_last_write_status", "RedisAofWriteFailed"),
("redis_aof_last_bgrewrite_status", "RedisAofRewriteFailed"),
]:
case(f"{metric} ok is one", alert, [series(metric, "1+0x30")], False)
case(f"{metric} failure is zero", alert, [series(metric, "0+0x30")], True)
for error in ["OOM", "MISCONF"]:
labels = {**base, "err": error}
case(f"new {error} counter starts at one", "RedisWriteRejected",
[series("redis_errors_total", "_ _ _ _ 1+0x26", labels)],
True, labels, at="3m")
case(f"old {error} error does not alert forever", "RedisWriteRejected",
[series("redis_errors_total", "1+0x30", labels)],
False, at="10m")
case("growing OOM error counter", "RedisWriteRejected",
[series("redis_errors_total", "0+1x30", {**base, "err": "OOM"})],
True, {**base, "err": "OOM"})
case("Redis memory high", "RedisMemoryNearMaxmemory", [
series("redis_memory_used_bytes", "3006477107+0x30"),
series("redis_mem_not_counted_for_eviction_bytes", "0+0x30"),
series("redis_memory_max_bytes", "3221225472+0x30"),
], True)
case("exclude buffers not counted by eviction", "RedisMemoryNearMaxmemory", [
series("redis_memory_used_bytes", "3221225472+0x30"),
series("redis_mem_not_counted_for_eviction_bytes", "1073741824+0x30"),
series("redis_memory_max_bytes", "3221225472+0x30"),
], False)
case("container limit evaluated independently from maxmemory", "RedisContainerMemoryHigh", [
series("container_memory_working_set_bytes", "4831838208+0x30", container),
series("kube_pod_container_resource_limits", "5368709120+0x30",
{**container, "resource": "memory"}),
], True, container)
for restarts, firing in [("0+1x30", True), ("1+0x30", False)]:
case(f"OOM reason with restarts {restarts}", "RedisContainerOomKilled", [
series("kube_pod_container_status_last_terminated_reason", "1+0x30",
{**container, "reason": "OOMKilled"}),
series("kube_pod_container_status_restarts_total", restarts, container),
], firing, container, at="12m")
for metric, alert in [
("redis_aof_delayed_fsync", "RedisAofFsyncDelayed"),
("redis_evicted_keys_total", "RedisUnexpectedEviction"),
]:
case(f"increasing {metric}", alert, [series(metric, "0+1x30")], True)
case(f"stable {metric}", alert, [series(metric, "1+0x30")], False)
disk = {"job": "node-exporter", "instance": "10.250.0.16:9100",
"mountpoint": "/", "fstype": "ext4", "device": "/dev/sda1"}
for node, free, firing in [
("heize", "9663676416", True),
("heize", "12884901888", False),
("suga", "9663676416", False),
]:
case(f"{node} free disk {free}", "RedisNodeDiskHeadroomLow", [
series("node_filesystem_avail_bytes", free + "+0x30", disk),
series("node_uname_info", "1+0x30",
{"job": "node-exporter", "instance": disk["instance"], "nodename": node}),
], firing, {**disk, "nodename": node})
with tempfile.TemporaryDirectory(prefix="redis-monitoring-") as directory:
rule_path = Path(directory) / "rules.json"
rule_path.write_text(json.dumps(rule_resource["spec"]), encoding="utf-8")
test_path = Path(directory) / "tests.json"
test_path.write_text(json.dumps({
"rule_files": [rule_path.as_posix()],
"evaluation_interval": "30s", "tests": tests,
}), encoding="utf-8")
subprocess.run([args.promtool, "check", "rules", str(rule_path)], check=True)
subprocess.run([args.promtool, "test", "rules", str(test_path)], check=True)
print(f"PASS: resource assertions and {len(tests)} alert scenarios")
if __name__ == "__main__":
main()
@@ -0,0 +1,42 @@
# Preserve the existing names so Argo CD can adopt the shared policies in place.
# These policies apply to all services in tekton-ci, not only lolympus.
apiVersion: v1
kind: ResourceQuota
metadata:
name: lolympus-ci
namespace: tekton-ci
labels:
app.kubernetes.io/name: tekton-ci
app.kubernetes.io/part-of: tekton-ci
app.kubernetes.io/managed-by: argocd
spec:
hard:
pods: "20"
# Reserve room for new runs between pruning the completed history to 10 runs.
persistentvolumeclaims: "15"
# Tekton steps run sequentially, but quota sums their container limits.
# Bound compute requests here; retain per-container limits in Tasks/LimitRange.
requests.cpu: "8"
requests.memory: 16Gi
requests.storage: 150Gi
count/pipelineruns.tekton.dev: "15"
count/taskruns.tekton.dev: "150"
---
apiVersion: v1
kind: LimitRange
metadata:
name: lolympus-ci
namespace: tekton-ci
labels:
app.kubernetes.io/name: tekton-ci
app.kubernetes.io/part-of: tekton-ci
app.kubernetes.io/managed-by: argocd
spec:
limits:
- type: Container
defaultRequest:
cpu: 100m
memory: 256Mi
default:
cpu: "2"
memory: 4Gi
@@ -0,0 +1,111 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: lolympus-tekton-eventlistener-interceptors
rules:
- apiGroups:
- triggers.tekton.dev
resources:
- clusterinterceptors
- clustertriggerbindings
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: lolympus-tekton-eventlistener-interceptors
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: lolympus-tekton-eventlistener-interceptors
subjects:
- kind: ServiceAccount
name: lolympus-event-listener
namespace: tekton-ci
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: url-shortener-tekton-eventlistener-interceptors
rules:
- apiGroups:
- triggers.tekton.dev
resources:
- clusterinterceptors
- clustertriggerbindings
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: url-shortener-tekton-eventlistener-interceptors
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: url-shortener-tekton-eventlistener-interceptors
subjects:
- kind: ServiceAccount
name: url-shortener-event-listener
namespace: tekton-ci
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: make-space-tekton-eventlistener-interceptors
rules:
- apiGroups:
- triggers.tekton.dev
resources:
- clusterinterceptors
- clustertriggerbindings
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: make-space-tekton-eventlistener-interceptors
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: make-space-tekton-eventlistener-interceptors
subjects:
- kind: ServiceAccount
name: make-space-event-listener
namespace: tekton-ci
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: blog-tekton-eventlistener-interceptors
rules:
- apiGroups:
- triggers.tekton.dev
resources:
- clusterinterceptors
- clustertriggerbindings
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: blog-tekton-eventlistener-interceptors
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: blog-tekton-eventlistener-interceptors
subjects:
- kind: ServiceAccount
name: blog-event-listener
namespace: tekton-ci
@@ -3,3 +3,5 @@ kind: Kustomization
resources: resources:
- tektonconfig.yaml - tektonconfig.yaml
- ingress.yaml - ingress.yaml
- eventlistener-rbac.yaml
- ci-resource-policy.yaml
+4 -4
View File
@@ -6,12 +6,12 @@ spec:
targetNamespace: tekton-pipelines targetNamespace: tekton-pipelines
profile: all profile: all
dashboard: dashboard:
readonly: true readonly: false
pruner: pruner:
disabled: false disabled: false
schedule: "0 */6 * * *" schedule: "*/15 * * * *"
resources: resources:
- taskrun
- pipelinerun - pipelinerun
# Keep the newest completed runs across each namespace; child resources follow their owner.
keep: 10 keep: 10
prune-per-resource: true # prune-per-resource defaults to false; the operator omits an explicit false.