update helm charts

charts/helm/checkmate/INSTALLATION.md

@@ -42,6 +42,71 @@ kubectl get svc
 
 Once all pods are `Running` and `Ready`, you can access Checkmate via the configured ingress hosts.
 
+## Scaling: the worker tier & autoscaling
+
+By default the chart runs a single all-in-one API pod that both schedules **and** processes monitoring
+jobs (`worker.enabled: false`). For larger deployments you can split processing onto a dedicated,
+horizontally-scalable worker tier.
+
+### Topology
+
+| `worker.enabled` | API (`checkmate-api`) | Worker (`checkmate-worker`) |
+| --- | --- | --- |
+| `false` (default) | schedules **and** processes jobs (`QUEUE_PRIMARY_PROCESSES=true`) | not deployed |
+| `true` | schedules + serves the API only (`QUEUE_PRIMARY_PROCESSES=false`, derived) | processes all jobs |
+
+`QUEUE_PRIMARY_PROCESSES` is **derived** from `worker.enabled` — you never set it directly, so the API
+and worker tier can never both process (or both ignore) the queue.
+
+### Enabling the worker tier
+
+```yaml
+worker:
+  enabled: true
+  replicas: 2   # used only when autoscaling.enabled = false
+```
+
+The worker takes **no inbound traffic** (no Service): the kubelet probes pod IPs directly on
+`HEALTH_PORT` (`/livez`, `/readyz`), and graceful shutdown drains in-flight jobs before exit.
+
+> **Invariant:** `worker.terminationGracePeriodSeconds` (default `30`) **must stay greater than the
+> server drain timeout (25s)**. Lower it and Kubernetes SIGKILLs workers mid-drain, losing in-flight
+> checks.
+
+> **Caveat:** with the worker tier enabled the API no longer processes jobs, so if **all** workers are
+> down nothing processes. The `autoscaling.minReplicaCount: 1` floor plus liveness restarts mitigate
+> this; full failover to the API would need leader election (not supported).
+
+### Autoscaling with KEDA
+
+Backlog-driven autoscaling is **off by default** and requires the
+[KEDA operator](https://keda.sh/) installed in-cluster. Enable it with:
+
+```yaml
+worker:
+  enabled: true
+  autoscaling:
+    enabled: true
+    minReplicaCount: 1     # processing floor — never 0, or checks silently stall
+    maxReplicaCount: 10
+    backlogPerReplica: 50  # target due-job backlog per worker
+    mongo:
+      dbName: uptime_db    # MUST match the database in DB_CONNECTION_STRING
+```
+
+A KEDA `ScaledObject` queries MongoDB directly for the due-check backlog and scales the worker
+Deployment between `minReplicaCount` and `maxReplicaCount`. When autoscaling is on, the Deployment
+omits `replicas` so it doesn't fight the HPA KEDA creates. Make sure `autoscaling.mongo.dbName`
+matches the database name in your `DB_CONNECTION_STRING`, or the scaler counts the wrong DB (always 0
+backlog → never scales up).
+
+#### Fallback: Prometheus-adapter HPA (not shipped)
+
+If your KEDA/Mongo versions reject the `$$NOW`/`$toLong` query, or you already run Prometheus, scrape
+the worker's `/metrics` (`checkmate_worker_due_backlog`), expose it via prometheus-adapter, and target
+it with a standard `HorizontalPodAutoscaler`. The manifest is left to the operator; the app computes
+`now` in code on this path, so the query coercion is unnecessary.
+
 ## Enabling TLS/HTTPS with cert-manager
 
 If you have [cert-manager](https://cert-manager.io/) installed in your cluster, you can enable automatic TLS certificate provisioning using Let's Encrypt or other certificate issuers.

charts/helm/checkmate/templates/api/deployment.yaml

@@ -23,6 +23,7 @@ spec:
       containers:
         - name: api
           image: {{ .Values.api.image }}
+          imagePullPolicy: {{ .Values.api.imagePullPolicy }}
           ports:
             - name: http
               containerPort: {{ .Values.api.port }}
@@ -57,5 +58,7 @@ spec:
             value: "{{ .Values.client.protocol }}://{{ .Values.client.ingress.host }}"
           - name: QUEUE_MODE
             value: "primary"
-          - name: QUEUE_PRIMARY_PROCESSES # false: API schedules only, worker tier processes jobs
-            value: {{ .Values.api.queuePrimaryProcesses | quote }}
+          # Derived: when a worker tier is deployed the API only schedules (false); otherwise it is
+          # all-in-one and processes jobs itself (true). Keeps the two from ever competing for jobs.
+          - name: QUEUE_PRIMARY_PROCESSES
+            value: {{ not .Values.worker.enabled | quote }}

charts/helm/checkmate/templates/client/deployment.yaml

@@ -23,6 +23,7 @@ spec:
       containers:
         - name: client
           image: {{ .Values.client.image }}
+          imagePullPolicy: {{ .Values.client.imagePullPolicy }}
           ports:
             - containerPort: {{ .Values.client.port }}
           env:

charts/helm/checkmate/templates/db/mongodb-statefulsets.yaml

@@ -23,6 +23,7 @@ spec:
       containers:
         - name: mongodb
           image: {{ .Values.mongodb.image }}
+          imagePullPolicy: {{ .Values.mongodb.imagePullPolicy }}
           ports:
             - containerPort: {{ .Values.mongodb.port }}
           command: ["mongod", "--quiet", "--bind_ip_all"]
@@ -37,7 +38,9 @@ spec:
     - metadata:
         name: checkmate-mongo-persistent-storage
       spec:
-        storageClassName: {{ .Values.persistence.mongo.storageClass | quote }}
+        {{- with .Values.persistence.mongo.storageClass }}
+        storageClassName: {{ . | quote }} {{/* omitted when empty so the cluster's default StorageClass applies */}}
+        {{- end }}
         accessModes: ["ReadWriteOnce"]
         resources:
           requests:

charts/helm/checkmate/templates/prechecks.yaml

@@ -45,4 +45,9 @@
   {{- fail "api.ingress.tls.enabled is true but no cert-manager issuer annotation found. Add 'cert-manager.io/cluster-issuer' or 'cert-manager.io/issuer'." }}
 {{- end }}
 
+{{/* Worker grace period must exceed the server drain timeout (25s) or workers are SIGKILLed mid-drain */}}
+{{- if and .Values.worker.enabled (le (int .Values.worker.terminationGracePeriodSeconds) 25) }}
+  {{- fail "worker.terminationGracePeriodSeconds must be greater than 25 (the server drain timeout) or workers are killed mid-drain" }}
+{{- end }}
+
 {{/* Secret name can be omitted; we default to <release>-client|api-tls in templates */}}

charts/helm/checkmate/templates/worker/deployment.yaml

@@ -0,0 +1,65 @@
+{{- if .Values.worker.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: checkmate-worker
+spec:
+  {{- if not .Values.worker.autoscaling.enabled }}
+  replicas: {{ .Values.worker.replicas }} # omitted when autoscaling owns the replica count
+  {{- end }}
+  selector:
+    matchLabels:
+      app: checkmate-worker
+  template:
+    metadata:
+      labels:
+        app: checkmate-worker
+    spec:
+      terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }}
+    {{- with .Values.worker.affinity }}
+      affinity:
+        {{- tpl ( . | toYaml) $ | nindent 8 }}
+    {{- end }}
+    {{- with .Values.worker.tolerations }}
+      tolerations:
+        {{- tpl ( . | toYaml) $ | nindent 8 }}
+    {{- end }}
+      containers:
+        - name: worker
+          image: {{ .Values.worker.image | default .Values.api.image }}
+          imagePullPolicy: {{ .Values.worker.imagePullPolicy | default .Values.api.imagePullPolicy }}
+          ports:
+            - name: health
+              containerPort: {{ .Values.worker.healthPort }}
+          envFrom:
+            - secretRef:
+                name: checkmate-secrets # DB_CONNECTION_STRING (+ JWT_SECRET/CLIENT_HOST for validateEnv)
+          env:
+          # Secret default CLIENT_HOST is the non-URL "change_me"; override like the API so validateEnv passes.
+          - name: CLIENT_HOST
+            value: "{{ .Values.client.protocol }}://{{ .Values.client.ingress.host }}"
+          - name: QUEUE_MODE
+            value: "worker"
+          - name: HEALTH_PORT
+            value: {{ .Values.worker.healthPort | quote }}
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: health
+            initialDelaySeconds: {{ .Values.worker.probes.liveness.initialDelaySeconds }}
+            periodSeconds: {{ .Values.worker.probes.liveness.periodSeconds }}
+            timeoutSeconds: {{ .Values.worker.probes.liveness.timeoutSeconds }}
+            failureThreshold: {{ .Values.worker.probes.liveness.failureThreshold }}
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: health
+            initialDelaySeconds: {{ .Values.worker.probes.readiness.initialDelaySeconds }}
+            periodSeconds: {{ .Values.worker.probes.readiness.periodSeconds }}
+            timeoutSeconds: {{ .Values.worker.probes.readiness.timeoutSeconds }}
+            failureThreshold: {{ .Values.worker.probes.readiness.failureThreshold }}
+          {{- with .Values.worker.resources }}
+          resources:
+              {{- tpl ( . | toYaml) $ | nindent 12 }}
+          {{- end }}
+{{- end }}

charts/helm/checkmate/templates/worker/scaledobject.yaml

@@ -0,0 +1,30 @@
+{{- if and .Values.worker.enabled .Values.worker.autoscaling.enabled }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: checkmate-worker
+spec:
+  scaleTargetRef:
+    name: checkmate-worker
+  minReplicaCount: {{ .Values.worker.autoscaling.minReplicaCount }} # floor: never 0 — a cold gap stops checks
+  maxReplicaCount: {{ .Values.worker.autoscaling.maxReplicaCount }}
+  cooldownPeriod: {{ .Values.worker.autoscaling.cooldownPeriod }}
+  advanced:
+    horizontalPodAutoscalerConfig:
+      behavior:
+        scaleDown:
+          stabilizationWindowSeconds: {{ .Values.worker.autoscaling.stabilizationWindowSeconds }}
+  triggers:
+    - type: mongodb
+      metadata:
+        dbName: {{ .Values.worker.autoscaling.mongo.dbName | quote }}
+        collection: {{ .Values.worker.autoscaling.mongo.collection | quote }}
+        # The Phase 2 countDueBacklog predicate, time-relativized with $$NOW (epoch-ms via $toLong):
+        query: >-
+          { "type": "check", "isActive": true,
+            "$expr": { "$lte": ["$nextScheduledAt", { "$toLong": "$$NOW" }] },
+            "$or": [ { "lockedUntil": null },
+                     { "$expr": { "$lt": ["$lockedUntil", { "$toLong": "$$NOW" }] } } ] }
+        queryValue: {{ .Values.worker.autoscaling.backlogPerReplica | quote }}
+        connectionStringFromEnv: DB_CONNECTION_STRING # resolved from the worker's checkmate-secrets
+{{- end }}

charts/helm/checkmate/values.yaml

@@ -1,5 +1,6 @@
 client:
   image: ghcr.io/bluewave-labs/checkmate-client:v3.2.0
+  imagePullPolicy: IfNotPresent # IfNotPresent so kind-loaded :latest images are used (not pulled)
   port: 80
   protocol: http
   ingress:
@@ -19,11 +20,9 @@ client:
 
 api:
   image: ghcr.io/bluewave-labs/checkmate-backend:v3.2.0
+  imagePullPolicy: IfNotPresent # IfNotPresent so kind-loaded :latest images are used (not pulled)
   port: 52345
   protocol: http
-  # false: API only schedules, the worker tier processes jobs (requires workers deployed).
-  # true: API also processes jobs (all-in-one; no separate worker needed).
-  queuePrimaryProcesses: false
   # Liveness/readiness probes hit GET /api/v1/health on the api port (proves the HTTP server is up).
   probes:
     liveness:
@@ -53,8 +52,44 @@ api:
 
 mongodb:
   image: ghcr.io/bluewave-labs/checkmate-mongo:v3.2.0
+  imagePullPolicy: IfNotPresent # IfNotPresent so kind-loaded :latest images are used (not pulled)
   port: 27017
 
+worker:
+  # Off by default; existing installs are unchanged. When true, a dedicated worker processes all
+  # jobs and the API stops processing (QUEUE_PRIMARY_PROCESSES is derived as false). When false, the
+  # API is all-in-one and processes jobs itself.
+  enabled: false
+  image: "" # defaults to api.image when empty, same as API image
+  imagePullPolicy: "" # defaults to api.imagePullPolicy when empty
+  replicas: 1 # used only when autoscaling.enabled = false
+  healthPort: 52346 # HEALTH_PORT; serves /livez /readyz /metrics
+  terminationGracePeriodSeconds: 30 # MUST stay > the server drain timeout (25s) or workers are killed mid-drain
+  resources: {}
+  affinity: {}
+  tolerations: {}
+  probes:
+    liveness: # /livez — failureThreshold>=3 so one slow tick never restarts mid-job
+      initialDelaySeconds: 10
+      periodSeconds: 10
+      timeoutSeconds: 3
+      failureThreshold: 3
+    readiness: # /readyz — flips to 503 on drain
+      initialDelaySeconds: 5
+      periodSeconds: 10
+      timeoutSeconds: 3
+      failureThreshold: 3
+  autoscaling:
+    enabled: false # requires the KEDA operator installed in-cluster
+    minReplicaCount: 1 # processing floor — never 0 (a cold gap silently stops checks)
+    maxReplicaCount: 10
+    backlogPerReplica: 50 # KEDA queryValue: target due-backlog per worker
+    cooldownPeriod: 300 # seconds before scaling an idle target down
+    stabilizationWindowSeconds: 300 # damp the interval-boundary herd
+    mongo:
+      dbName: uptime_db # must match DB_CONNECTION_STRING's database
+      collection: jobs
+
 secrets:
   JWT_SECRET: change_me
   DB_CONNECTION_STRING: mongodb://checkmate-mongodb:27017/uptime_db