sure: fix the readiness probe, the ephemeral-storage budget and HOME

Second install-test run, three findings, all measured:

- Readiness on "/" failed with "http: server gave HTTP response to HTTPS
  client" on a pod that was answering correctly. The kubelet's HTTP prober
  follows redirects to the same host, and with RAILS_ASSUME_SSL=true the
  app builds its redirect as https://<pod-ip>:3000/registration/new, which
  Puma does not speak. The probe now runs curl without -L and accepts
  2xx-3xx itself. The same log also proves what the probe is for: against
  an empty database the redirect target was /registration/new, the branch
  User.count decides, so the path really does reach Postgres.
- The pod was evicted within two minutes of every start: "ephemeral local
  storage usage exceeds the total limit of containers 60Mi". emptyDirs and
  the stdout log both count there, and the image rebuilds its Bootsnap
  cache under /rails/tmp on every start because the emptyDir hides the one
  baked into the image. The request is now 256Mi (limit 768Mi) per app
  container - headroom, not a measurement.
- /home/rails is read-only, so Bundler created a throwaway home under
  TMPDIR on every invocation, worker probe included. HOME now points at a
  directory the init container creates.

The CI postgres fixture gets -d sure on its readiness probe, so the server
stops logging a missing database that nothing asked for.

ci/sure/fixtures.yaml

@@ -64,8 +64,13 @@ spec:
           ports:
             - containerPort: 5432
           readinessProbe:
+            # -d names the database the chart actually uses. Without it
+            # pg_isready asks for a database named after the user and the
+            # server logs 'FATAL: database "ci" does not exist' every five
+            # seconds - noise that reads like a chart problem in the diagnostics
+            # dump when something else fails.
             exec:
-              command: ["pg_isready", "-U", "ci"]
+              command: ["pg_isready", "-U", "ci", "-d", "sure"]
             periodSeconds: 5
           volumeMounts:
             - name: data

sure/templates/_helpers.tpl

@@ -237,11 +237,15 @@ while the database is unreachable. The long startup budget covers the
 {{- include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.web.livenessProbe") -}}
 {{- end -}}
 
+{{- define "sure.webReadinessCommand" -}}
+{{- list "/bin/bash" "-c" "code=\"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:3000/)\"; test \"$code\" -ge 200 -a \"$code\" -lt 400" | toJson -}}
+{{- end -}}
+
 {{- define "sure.webReadinessProbe" -}}
 {{- $defaults := dict
-      "httpGet" (dict "path" "/" "port" "http")
+      "exec" (dict "command" (include "sure.webReadinessCommand" . | fromJsonArray))
       "periodSeconds" 10
-      "timeoutSeconds" 5
+      "timeoutSeconds" 10
       "failureThreshold" 3 -}}
 {{- include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.web.readinessProbe") -}}
 {{- end -}}
@@ -301,7 +305,7 @@ comes from.
 {{- define "sure.webResources" -}}
 {{- $defaults := dict
       "limitFactor" 3
-      "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "10Mi") -}}
+      "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "256Mi") -}}
 {{- $merged := fromYaml (include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.web.resources")) -}}
 {{- include "sure.resources" $merged -}}
 {{- end -}}
@@ -309,7 +313,7 @@ comes from.
 {{- define "sure.workerResources" -}}
 {{- $defaults := dict
       "limitFactor" 3
-      "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "10Mi") -}}
+      "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "256Mi") -}}
 {{- $merged := fromYaml (include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.worker.resources")) -}}
 {{- include "sure.resources" $merged -}}
 {{- end -}}
@@ -343,6 +347,15 @@ probe on "/" reach the database at all (see values.yaml).
 # init container in the StatefulSet, which explains the whole case.
 - name: TMPDIR
   value: "/tmp/ruby"
+# The image's home directory (/home/rails) is part of the read-only root
+# filesystem, and Bundler reacts to that by creating a throwaway home under
+# TMPDIR on EVERY invocation ("`/home/rails` is not writable. Bundler will use
+# `/tmp/ruby/bundler2026...` as your home directory temporarily", measured in
+# the install-test). That is one directory per `bundle exec` - including every
+# worker probe, every 30 seconds. Pointing HOME at a writable directory turns
+# that into one stable path.
+- name: HOME
+  value: "/tmp/home"
 - name: PORT
   value: "3000"
 # Sure's production.rb defaults BOTH of these to true. The chart renders them

sure/templates/sure.sfs.yaml

@@ -53,10 +53,12 @@ spec:
         # file yet.
         #
         # The fix is a directory that is writable without being world-writable.
-        # This init container creates one per app container inside the same
+        # This init container creates two of them per app container inside the
-        # emptyDirs (mode 0700, owned by the pod's uid) and both containers get
+        # same emptyDirs (mode 0700, owned by the pod's uid): "ruby" for TMPDIR,
-        # TMPDIR pointed at it. It runs as the same non-root uid as the app and
+        # and "home" for HOME - the image's /home/rails is read-only, and
-        # needs no capability at all: the emptyDir root is 0777, so creating an
+        # Bundler answers that by creating a throwaway home under TMPDIR on
+        # every single invocation. It runs as the same non-root uid as the app
+        # and needs no capability at all: the emptyDir root is 0777, so creating an
         # entry in it is allowed, and chmod on a directory one has just created
         # needs no privilege. That is deliberately weaker than the chown-as-root
         # init container paperless-ngx needs.
@@ -71,7 +73,7 @@ spec:
             - /bin/bash
             - -ec
             - |
-              for dir in /web-tmp/ruby /worker-tmp/ruby; do
+              for dir in /web-tmp/ruby /web-tmp/home /worker-tmp/ruby /worker-tmp/home; do
                 mkdir -p "$dir"
                 chmod 700 "$dir"
               done

sure/values.yaml

@@ -44,10 +44,12 @@ sure:
   # readOnlyRootFilesystem: true holds, and the writable paths are emptyDirs:
   # /rails/tmp, /rails/log and /tmp, one set per container. /tmp needs one extra
   # step that is easy to mistake for decoration - a tiny init container creates
-  # a private subdirectory in it and the containers run with TMPDIR pointing
+  # two private subdirectories in it, and the containers run with TMPDIR and
-  # there, because Ruby refuses a world-writable directory without a sticky bit
+  # HOME pointing there. Ruby refuses a world-writable directory without a
-  # as its temporary directory and an emptyDir is exactly that. Without it the
+  # sticky bit as its temporary directory and an emptyDir is exactly that, so
-  # app aborts during boot; the StatefulSet carries the measured evidence.
+  # without this the app aborts during boot; HOME follows because the image's
+  # /home/rails is read-only and Bundler otherwise builds a throwaway home per
+  # invocation. The StatefulSet carries the measured evidence for both.
   securityContext:
     pod:
       runAsNonRoot: true
@@ -89,11 +91,21 @@ sure:
       # x limitFactor 3 = 1536Mi limit, which holds both the everyday figure and
       # the AI recommendation without reserving the larger number on the node.
       # Read from upstream's documentation, not measured here.
+      #
+      # ephemeral-storage is NOT the template default of 10Mi, and the reason is
+      # measured: with 10Mi (limit 30Mi) per container the kubelet evicted the
+      # pod within two minutes of every start - "Pod ephemeral local storage
+      # usage exceeds the total limit of containers 60Mi". Both emptyDirs and
+      # the container's stdout log count against that limit, and this image
+      # rebuilds its Bootsnap cache under /rails/tmp on every pod start because
+      # the emptyDir hides the one baked into the image. 256Mi request (768Mi
+      # limit) is headroom for that, not a measurement of it - the steady-state
+      # figure is still unknown.
       limitFactor: 3
       requests:
         cpu: 0.1
         memory: 512Mi
-        ephemeral-storage: 10Mi
+        ephemeral-storage: 256Mi
     # PROBES - what each one actually proves (README, guideline 12.3)
     #
     # /up is Rails' own health route (config/routes.rb: get "up" =>
@@ -112,32 +124,41 @@ sure:
       periodSeconds: 10
       timeoutSeconds: 5
       failureThreshold: 3
-    # Readiness uses "/" instead, because that path provably reaches the
+    # Readiness asks for "/" instead, because that path provably reaches the
-    # database. Read from the source of 0.7.4, not measured: every request runs
+    # database - and it does so through curl rather than an httpGet probe, for
-    # ApplicationController's authenticate_user! before_action
+    # a reason that was measured rather than guessed (see below).
-    # (app/controllers/concerns/authentication.rb); without a session cookie -
-    # which a kubelet probe never has - it falls into self_hosted_first_login?,
-    # which is "config.app_mode.self_hosted? && User.count.zero?". User.count is
-    # an SQL query. With the database up the request ends in a 302 to the login
-    # or registration page, which the kubelet counts as success (2xx-3xx); with
-    # the database down the query raises and Rails answers 500.
     #
-    # Two conditions this depends on, both of them chart-set:
+    # Why "/" reaches the database: every request runs ApplicationController's
-    #   - SELF_HOSTED=true. In managed mode the same code short-circuits before
+    # authenticate_user! before_action (app/controllers/concerns/
-    #     User.count and "/" would redirect without touching Postgres.
+    # authentication.rb); without a session cookie - which a probe never has -
-    #   - the probe must not be answered by a redirect BEFORE the controller.
+    # it falls into self_hosted_first_login?, i.e.
-    #     Sure's production.rb defaults RAILS_FORCE_SSL to true, and a forced
+    # "config.app_mode.self_hosted? && User.count.zero?". User.count is an SQL
-    #     redirect to https is a 301 - i.e. a "success" that proves nothing.
+    # query. Measured in the install-test: against an empty database the app
-    #     The chart sets RAILS_ASSUME_SSL alongside it (see the env block that
+    # answered 302 to /registration/new, which is the branch that query decides,
-    #     the templates always render), so requests are treated as SSL and no
+    # so the path demonstrably touches Postgres. With the database down the
-    #     redirect happens. Overriding RAILS_ASSUME_SSL to "false" while
+    # query raises and Rails answers 500.
-    #     RAILS_FORCE_SSL stays "true" turns this probe back into a lie.
+    #
+    # Why exec + curl and not httpGet: the kubelet's HTTP prober FOLLOWS
+    # redirects to the same host, and with RAILS_ASSUME_SSL=true (which the
+    # chart sets whenever the Ingress is on) Rails builds that redirect as
+    # https://<pod-ip>:3000/... - a scheme Puma does not speak. The install-test
+    # failed with "http: server gave HTTP response to HTTPS client" on a pod
+    # that was answering correctly. curl without -L asks once and reports the
+    # status code; the test accepts 2xx and 3xx exactly as the kubelet would,
+    # and a 500 from a missing database still fails. curl is part of the image
+    # (the Dockerfile installs it), so nothing extra is pulled in for this.
+    #
+    # One condition this depends on: SELF_HOSTED=true, which the chart always
+    # sets. In managed mode the same code short-circuits before User.count and
+    # "/" would redirect without touching Postgres at all.
     readinessProbe:
-      httpGet:
+      exec:
-        path: /
+        command:
-        port: http
+          - /bin/bash
+          - -c
+          - "code=\"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:3000/)\"; test \"$code\" -ge 200 -a \"$code\" -lt 400"
       periodSeconds: 10
-      timeoutSeconds: 5
+      timeoutSeconds: 10
       failureThreshold: 3
     # The startup probe gates liveness and readiness, so neither needs an
     # initialDelaySeconds. Its budget has to cover the first start, which is the
@@ -191,11 +212,14 @@ sure:
       # one operation that deterministically exceeds 512 MB (observed flat at
       # ~680 MB on a 2 GB container). The computed limit of 3 x 512Mi = 1536Mi
       # covers that; the request stays at the everyday figure.
+      #
+      # ephemeral-storage: same 256Mi as the web container and for the same
+      # measured reason - see the comment there.
       limitFactor: 3
       requests:
         cpu: 0.1
         memory: 512Mi
-        ephemeral-storage: 10Mi
+        ephemeral-storage: 256Mi
     # PROBES - the worker speaks no HTTP at all, so all three are exec probes.
     #
     # The command asks Sidekiq's own monitor whether THIS process is in the