diff --git a/charts/helm/checkmate/INSTALLATION.md b/charts/helm/checkmate/INSTALLATION.md index ae0f36b8b..c0528b66b 100644 --- a/charts/helm/checkmate/INSTALLATION.md +++ b/charts/helm/checkmate/INSTALLATION.md @@ -42,6 +42,71 @@ kubectl get svc Once all pods are `Running` and `Ready`, you can access Checkmate via the configured ingress hosts. +## Scaling: the worker tier & autoscaling + +By default the chart runs a single all-in-one API pod that both schedules **and** processes monitoring +jobs (`worker.enabled: false`). For larger deployments you can split processing onto a dedicated, +horizontally-scalable worker tier. + +### Topology + +| `worker.enabled` | API (`checkmate-api`) | Worker (`checkmate-worker`) | +| --- | --- | --- | +| `false` (default) | schedules **and** processes jobs (`QUEUE_PRIMARY_PROCESSES=true`) | not deployed | +| `true` | schedules + serves the API only (`QUEUE_PRIMARY_PROCESSES=false`, derived) | processes all jobs | + +`QUEUE_PRIMARY_PROCESSES` is **derived** from `worker.enabled` — you never set it directly, so the API +and worker tier can never both process (or both ignore) the queue. + +### Enabling the worker tier + +```yaml +worker: + enabled: true + replicas: 2 # used only when autoscaling.enabled = false +``` + +The worker takes **no inbound traffic** (no Service): the kubelet probes pod IPs directly on +`HEALTH_PORT` (`/livez`, `/readyz`), and graceful shutdown drains in-flight jobs before exit. + +> **Invariant:** `worker.terminationGracePeriodSeconds` (default `30`) **must stay greater than the +> server drain timeout (25s)**. Lower it and Kubernetes SIGKILLs workers mid-drain, losing in-flight +> checks. + +> **Caveat:** with the worker tier enabled the API no longer processes jobs, so if **all** workers are +> down nothing processes. The `autoscaling.minReplicaCount: 1` floor plus liveness restarts mitigate +> this; full failover to the API would need leader election (not supported). + +### Autoscaling with KEDA + +Backlog-driven autoscaling is **off by default** and requires the +[KEDA operator](https://keda.sh/) installed in-cluster. Enable it with: + +```yaml +worker: + enabled: true + autoscaling: + enabled: true + minReplicaCount: 1 # processing floor — never 0, or checks silently stall + maxReplicaCount: 10 + backlogPerReplica: 50 # target due-job backlog per worker + mongo: + dbName: uptime_db # MUST match the database in DB_CONNECTION_STRING +``` + +A KEDA `ScaledObject` queries MongoDB directly for the due-check backlog and scales the worker +Deployment between `minReplicaCount` and `maxReplicaCount`. When autoscaling is on, the Deployment +omits `replicas` so it doesn't fight the HPA KEDA creates. Make sure `autoscaling.mongo.dbName` +matches the database name in your `DB_CONNECTION_STRING`, or the scaler counts the wrong DB (always 0 +backlog → never scales up). + +#### Fallback: Prometheus-adapter HPA (not shipped) + +If your KEDA/Mongo versions reject the `$$NOW`/`$toLong` query, or you already run Prometheus, scrape +the worker's `/metrics` (`checkmate_worker_due_backlog`), expose it via prometheus-adapter, and target +it with a standard `HorizontalPodAutoscaler`. The manifest is left to the operator; the app computes +`now` in code on this path, so the query coercion is unnecessary. + ## Enabling TLS/HTTPS with cert-manager If you have [cert-manager](https://cert-manager.io/) installed in your cluster, you can enable automatic TLS certificate provisioning using Let's Encrypt or other certificate issuers. diff --git a/charts/helm/checkmate/templates/api/deployment.yaml b/charts/helm/checkmate/templates/api/deployment.yaml index ec2e05d99..8433ecd58 100644 --- a/charts/helm/checkmate/templates/api/deployment.yaml +++ b/charts/helm/checkmate/templates/api/deployment.yaml @@ -23,6 +23,7 @@ spec: containers: - name: api image: {{ .Values.api.image }} + imagePullPolicy: {{ .Values.api.imagePullPolicy }} ports: - name: http containerPort: {{ .Values.api.port }} @@ -57,5 +58,7 @@ spec: value: "{{ .Values.client.protocol }}://{{ .Values.client.ingress.host }}" - name: QUEUE_MODE value: "primary" - - name: QUEUE_PRIMARY_PROCESSES # false: API schedules only, worker tier processes jobs - value: {{ .Values.api.queuePrimaryProcesses | quote }} + # Derived: when a worker tier is deployed the API only schedules (false); otherwise it is + # all-in-one and processes jobs itself (true). Keeps the two from ever competing for jobs. + - name: QUEUE_PRIMARY_PROCESSES + value: {{ not .Values.worker.enabled | quote }} diff --git a/charts/helm/checkmate/templates/client/deployment.yaml b/charts/helm/checkmate/templates/client/deployment.yaml index b59d3a4a9..2919dbd4d 100644 --- a/charts/helm/checkmate/templates/client/deployment.yaml +++ b/charts/helm/checkmate/templates/client/deployment.yaml @@ -23,6 +23,7 @@ spec: containers: - name: client image: {{ .Values.client.image }} + imagePullPolicy: {{ .Values.client.imagePullPolicy }} ports: - containerPort: {{ .Values.client.port }} env: diff --git a/charts/helm/checkmate/templates/db/mongodb-statefulsets.yaml b/charts/helm/checkmate/templates/db/mongodb-statefulsets.yaml index 8149b77e2..d4e048ee7 100644 --- a/charts/helm/checkmate/templates/db/mongodb-statefulsets.yaml +++ b/charts/helm/checkmate/templates/db/mongodb-statefulsets.yaml @@ -23,6 +23,7 @@ spec: containers: - name: mongodb image: {{ .Values.mongodb.image }} + imagePullPolicy: {{ .Values.mongodb.imagePullPolicy }} ports: - containerPort: {{ .Values.mongodb.port }} command: ["mongod", "--quiet", "--bind_ip_all"] @@ -37,7 +38,9 @@ spec: - metadata: name: checkmate-mongo-persistent-storage spec: - storageClassName: {{ .Values.persistence.mongo.storageClass | quote }} + {{- with .Values.persistence.mongo.storageClass }} + storageClassName: {{ . | quote }} {{/* omitted when empty so the cluster's default StorageClass applies */}} + {{- end }} accessModes: ["ReadWriteOnce"] resources: requests: diff --git a/charts/helm/checkmate/templates/prechecks.yaml b/charts/helm/checkmate/templates/prechecks.yaml index 4cd337db3..b8ff2ef11 100644 --- a/charts/helm/checkmate/templates/prechecks.yaml +++ b/charts/helm/checkmate/templates/prechecks.yaml @@ -45,4 +45,9 @@ {{- fail "api.ingress.tls.enabled is true but no cert-manager issuer annotation found. Add 'cert-manager.io/cluster-issuer' or 'cert-manager.io/issuer'." }} {{- end }} +{{/* Worker grace period must exceed the server drain timeout (25s) or workers are SIGKILLed mid-drain */}} +{{- if and .Values.worker.enabled (le (int .Values.worker.terminationGracePeriodSeconds) 25) }} + {{- fail "worker.terminationGracePeriodSeconds must be greater than 25 (the server drain timeout) or workers are killed mid-drain" }} +{{- end }} + {{/* Secret name can be omitted; we default to -client|api-tls in templates */}} diff --git a/charts/helm/checkmate/templates/worker/deployment.yaml b/charts/helm/checkmate/templates/worker/deployment.yaml new file mode 100644 index 000000000..a4a76b541 --- /dev/null +++ b/charts/helm/checkmate/templates/worker/deployment.yaml @@ -0,0 +1,65 @@ +{{- if .Values.worker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: checkmate-worker +spec: + {{- if not .Values.worker.autoscaling.enabled }} + replicas: {{ .Values.worker.replicas }} # omitted when autoscaling owns the replica count + {{- end }} + selector: + matchLabels: + app: checkmate-worker + template: + metadata: + labels: + app: checkmate-worker + spec: + terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }} + {{- with .Values.worker.affinity }} + affinity: + {{- tpl ( . | toYaml) $ | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- tpl ( . | toYaml) $ | nindent 8 }} + {{- end }} + containers: + - name: worker + image: {{ .Values.worker.image | default .Values.api.image }} + imagePullPolicy: {{ .Values.worker.imagePullPolicy | default .Values.api.imagePullPolicy }} + ports: + - name: health + containerPort: {{ .Values.worker.healthPort }} + envFrom: + - secretRef: + name: checkmate-secrets # DB_CONNECTION_STRING (+ JWT_SECRET/CLIENT_HOST for validateEnv) + env: + # Secret default CLIENT_HOST is the non-URL "change_me"; override like the API so validateEnv passes. + - name: CLIENT_HOST + value: "{{ .Values.client.protocol }}://{{ .Values.client.ingress.host }}" + - name: QUEUE_MODE + value: "worker" + - name: HEALTH_PORT + value: {{ .Values.worker.healthPort | quote }} + livenessProbe: + httpGet: + path: /livez + port: health + initialDelaySeconds: {{ .Values.worker.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.worker.probes.liveness.timeoutSeconds }} + failureThreshold: {{ .Values.worker.probes.liveness.failureThreshold }} + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: {{ .Values.worker.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.worker.probes.readiness.timeoutSeconds }} + failureThreshold: {{ .Values.worker.probes.readiness.failureThreshold }} + {{- with .Values.worker.resources }} + resources: + {{- tpl ( . | toYaml) $ | nindent 12 }} + {{- end }} +{{- end }} diff --git a/charts/helm/checkmate/templates/worker/scaledobject.yaml b/charts/helm/checkmate/templates/worker/scaledobject.yaml new file mode 100644 index 000000000..09fc9a8a3 --- /dev/null +++ b/charts/helm/checkmate/templates/worker/scaledobject.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.worker.enabled .Values.worker.autoscaling.enabled }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: checkmate-worker +spec: + scaleTargetRef: + name: checkmate-worker + minReplicaCount: {{ .Values.worker.autoscaling.minReplicaCount }} # floor: never 0 — a cold gap stops checks + maxReplicaCount: {{ .Values.worker.autoscaling.maxReplicaCount }} + cooldownPeriod: {{ .Values.worker.autoscaling.cooldownPeriod }} + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: {{ .Values.worker.autoscaling.stabilizationWindowSeconds }} + triggers: + - type: mongodb + metadata: + dbName: {{ .Values.worker.autoscaling.mongo.dbName | quote }} + collection: {{ .Values.worker.autoscaling.mongo.collection | quote }} + # The Phase 2 countDueBacklog predicate, time-relativized with $$NOW (epoch-ms via $toLong): + query: >- + { "type": "check", "isActive": true, + "$expr": { "$lte": ["$nextScheduledAt", { "$toLong": "$$NOW" }] }, + "$or": [ { "lockedUntil": null }, + { "$expr": { "$lt": ["$lockedUntil", { "$toLong": "$$NOW" }] } } ] } + queryValue: {{ .Values.worker.autoscaling.backlogPerReplica | quote }} + connectionStringFromEnv: DB_CONNECTION_STRING # resolved from the worker's checkmate-secrets +{{- end }} diff --git a/charts/helm/checkmate/values.yaml b/charts/helm/checkmate/values.yaml index 42c98f73c..7f8fbc65d 100644 --- a/charts/helm/checkmate/values.yaml +++ b/charts/helm/checkmate/values.yaml @@ -1,5 +1,6 @@ client: image: ghcr.io/bluewave-labs/checkmate-client:v3.2.0 + imagePullPolicy: IfNotPresent # IfNotPresent so kind-loaded :latest images are used (not pulled) port: 80 protocol: http ingress: @@ -19,11 +20,9 @@ client: api: image: ghcr.io/bluewave-labs/checkmate-backend:v3.2.0 + imagePullPolicy: IfNotPresent # IfNotPresent so kind-loaded :latest images are used (not pulled) port: 52345 protocol: http - # false: API only schedules, the worker tier processes jobs (requires workers deployed). - # true: API also processes jobs (all-in-one; no separate worker needed). - queuePrimaryProcesses: false # Liveness/readiness probes hit GET /api/v1/health on the api port (proves the HTTP server is up). probes: liveness: @@ -53,8 +52,44 @@ api: mongodb: image: ghcr.io/bluewave-labs/checkmate-mongo:v3.2.0 + imagePullPolicy: IfNotPresent # IfNotPresent so kind-loaded :latest images are used (not pulled) port: 27017 +worker: + # Off by default; existing installs are unchanged. When true, a dedicated worker processes all + # jobs and the API stops processing (QUEUE_PRIMARY_PROCESSES is derived as false). When false, the + # API is all-in-one and processes jobs itself. + enabled: false + image: "" # defaults to api.image when empty, same as API image + imagePullPolicy: "" # defaults to api.imagePullPolicy when empty + replicas: 1 # used only when autoscaling.enabled = false + healthPort: 52346 # HEALTH_PORT; serves /livez /readyz /metrics + terminationGracePeriodSeconds: 30 # MUST stay > the server drain timeout (25s) or workers are killed mid-drain + resources: {} + affinity: {} + tolerations: {} + probes: + liveness: # /livez — failureThreshold>=3 so one slow tick never restarts mid-job + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readiness: # /readyz — flips to 503 on drain + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + autoscaling: + enabled: false # requires the KEDA operator installed in-cluster + minReplicaCount: 1 # processing floor — never 0 (a cold gap silently stops checks) + maxReplicaCount: 10 + backlogPerReplica: 50 # KEDA queryValue: target due-backlog per worker + cooldownPeriod: 300 # seconds before scaling an idle target down + stabilizationWindowSeconds: 300 # damp the interval-boundary herd + mongo: + dbName: uptime_db # must match DB_CONNECTION_STRING's database + collection: jobs + secrets: JWT_SECRET: change_me DB_CONNECTION_STRING: mongodb://checkmate-mongodb:27017/uptime_db