From d9267b21802a41677fa2380661936c56211b3ca5 Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 22 Sep 2026 20:43:53 +0000 Subject: [PATCH] sure: fix the readiness probe, the ephemeral-storage budget and HOME Second install-test run, three findings, all measured: - Readiness on "/" failed with "http: server gave HTTP response to HTTPS client" on a pod that was answering correctly. The kubelet's HTTP prober follows redirects to the same host, and with RAILS_ASSUME_SSL=true the app builds its redirect as https://:3000/registration/new, which Puma does not speak. The probe now runs curl without -L and accepts 2xx-3xx itself. The same log also proves what the probe is for: against an empty database the redirect target was /registration/new, the branch User.count decides, so the path really does reach Postgres. - The pod was evicted within two minutes of every start: "ephemeral local storage usage exceeds the total limit of containers 60Mi". emptyDirs and the stdout log both count there, and the image rebuilds its Bootsnap cache under /rails/tmp on every start because the emptyDir hides the one baked into the image. The request is now 256Mi (limit 768Mi) per app container - headroom, not a measurement. - /home/rails is read-only, so Bundler created a throwaway home under TMPDIR on every invocation, worker probe included. HOME now points at a directory the init container creates. The CI postgres fixture gets -d sure on its readiness probe, so the server stops logging a missing database that nothing asked for. --- ci/sure/fixtures.yaml | 7 ++- sure/templates/_helpers.tpl | 21 +++++++-- sure/templates/sure.sfs.yaml | 12 +++--- sure/values.yaml | 82 +++++++++++++++++++++++------------- 4 files changed, 83 insertions(+), 39 deletions(-) diff --git a/ci/sure/fixtures.yaml b/ci/sure/fixtures.yaml index 42d43fa..715e4e1 100644 --- a/ci/sure/fixtures.yaml +++ b/ci/sure/fixtures.yaml @@ -64,8 +64,13 @@ spec: ports: - containerPort: 5432 readinessProbe: + # -d names the database the chart actually uses. Without it + # pg_isready asks for a database named after the user and the + # server logs 'FATAL: database "ci" does not exist' every five + # seconds - noise that reads like a chart problem in the diagnostics + # dump when something else fails. exec: - command: ["pg_isready", "-U", "ci"] + command: ["pg_isready", "-U", "ci", "-d", "sure"] periodSeconds: 5 volumeMounts: - name: data diff --git a/sure/templates/_helpers.tpl b/sure/templates/_helpers.tpl index a86ea0a..03eef0b 100644 --- a/sure/templates/_helpers.tpl +++ b/sure/templates/_helpers.tpl @@ -237,11 +237,15 @@ while the database is unreachable. The long startup budget covers the {{- include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.web.livenessProbe") -}} {{- end -}} +{{- define "sure.webReadinessCommand" -}} +{{- list "/bin/bash" "-c" "code=\"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:3000/)\"; test \"$code\" -ge 200 -a \"$code\" -lt 400" | toJson -}} +{{- end -}} + {{- define "sure.webReadinessProbe" -}} {{- $defaults := dict - "httpGet" (dict "path" "/" "port" "http") + "exec" (dict "command" (include "sure.webReadinessCommand" . | fromJsonArray)) "periodSeconds" 10 - "timeoutSeconds" 5 + "timeoutSeconds" 10 "failureThreshold" 3 -}} {{- include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.web.readinessProbe") -}} {{- end -}} @@ -301,7 +305,7 @@ comes from. {{- define "sure.webResources" -}} {{- $defaults := dict "limitFactor" 3 - "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "10Mi") -}} + "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "256Mi") -}} {{- $merged := fromYaml (include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.web.resources")) -}} {{- include "sure.resources" $merged -}} {{- end -}} @@ -309,7 +313,7 @@ comes from. {{- define "sure.workerResources" -}} {{- $defaults := dict "limitFactor" 3 - "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "10Mi") -}} + "requests" (dict "cpu" 0.1 "memory" "512Mi" "ephemeral-storage" "256Mi") -}} {{- $merged := fromYaml (include "sure.defaultedDict" (dict "defaults" $defaults "given" . "path" "sure.worker.resources")) -}} {{- include "sure.resources" $merged -}} {{- end -}} @@ -343,6 +347,15 @@ probe on "/" reach the database at all (see values.yaml). # init container in the StatefulSet, which explains the whole case. - name: TMPDIR value: "/tmp/ruby" +# The image's home directory (/home/rails) is part of the read-only root +# filesystem, and Bundler reacts to that by creating a throwaway home under +# TMPDIR on EVERY invocation ("`/home/rails` is not writable. Bundler will use +# `/tmp/ruby/bundler2026...` as your home directory temporarily", measured in +# the install-test). That is one directory per `bundle exec` - including every +# worker probe, every 30 seconds. Pointing HOME at a writable directory turns +# that into one stable path. +- name: HOME + value: "/tmp/home" - name: PORT value: "3000" # Sure's production.rb defaults BOTH of these to true. The chart renders them diff --git a/sure/templates/sure.sfs.yaml b/sure/templates/sure.sfs.yaml index 27ba61f..b6f4b5c 100644 --- a/sure/templates/sure.sfs.yaml +++ b/sure/templates/sure.sfs.yaml @@ -53,10 +53,12 @@ spec: # file yet. # # The fix is a directory that is writable without being world-writable. - # This init container creates one per app container inside the same - # emptyDirs (mode 0700, owned by the pod's uid) and both containers get - # TMPDIR pointed at it. It runs as the same non-root uid as the app and - # needs no capability at all: the emptyDir root is 0777, so creating an + # This init container creates two of them per app container inside the + # same emptyDirs (mode 0700, owned by the pod's uid): "ruby" for TMPDIR, + # and "home" for HOME - the image's /home/rails is read-only, and + # Bundler answers that by creating a throwaway home under TMPDIR on + # every single invocation. It runs as the same non-root uid as the app + # and needs no capability at all: the emptyDir root is 0777, so creating an # entry in it is allowed, and chmod on a directory one has just created # needs no privilege. That is deliberately weaker than the chown-as-root # init container paperless-ngx needs. @@ -71,7 +73,7 @@ spec: - /bin/bash - -ec - | - for dir in /web-tmp/ruby /worker-tmp/ruby; do + for dir in /web-tmp/ruby /web-tmp/home /worker-tmp/ruby /worker-tmp/home; do mkdir -p "$dir" chmod 700 "$dir" done diff --git a/sure/values.yaml b/sure/values.yaml index b8eae6b..2b3acf2 100644 --- a/sure/values.yaml +++ b/sure/values.yaml @@ -44,10 +44,12 @@ sure: # readOnlyRootFilesystem: true holds, and the writable paths are emptyDirs: # /rails/tmp, /rails/log and /tmp, one set per container. /tmp needs one extra # step that is easy to mistake for decoration - a tiny init container creates - # a private subdirectory in it and the containers run with TMPDIR pointing - # there, because Ruby refuses a world-writable directory without a sticky bit - # as its temporary directory and an emptyDir is exactly that. Without it the - # app aborts during boot; the StatefulSet carries the measured evidence. + # two private subdirectories in it, and the containers run with TMPDIR and + # HOME pointing there. Ruby refuses a world-writable directory without a + # sticky bit as its temporary directory and an emptyDir is exactly that, so + # without this the app aborts during boot; HOME follows because the image's + # /home/rails is read-only and Bundler otherwise builds a throwaway home per + # invocation. The StatefulSet carries the measured evidence for both. securityContext: pod: runAsNonRoot: true @@ -89,11 +91,21 @@ sure: # x limitFactor 3 = 1536Mi limit, which holds both the everyday figure and # the AI recommendation without reserving the larger number on the node. # Read from upstream's documentation, not measured here. + # + # ephemeral-storage is NOT the template default of 10Mi, and the reason is + # measured: with 10Mi (limit 30Mi) per container the kubelet evicted the + # pod within two minutes of every start - "Pod ephemeral local storage + # usage exceeds the total limit of containers 60Mi". Both emptyDirs and + # the container's stdout log count against that limit, and this image + # rebuilds its Bootsnap cache under /rails/tmp on every pod start because + # the emptyDir hides the one baked into the image. 256Mi request (768Mi + # limit) is headroom for that, not a measurement of it - the steady-state + # figure is still unknown. limitFactor: 3 requests: cpu: 0.1 memory: 512Mi - ephemeral-storage: 10Mi + ephemeral-storage: 256Mi # PROBES - what each one actually proves (README, guideline 12.3) # # /up is Rails' own health route (config/routes.rb: get "up" => @@ -112,32 +124,41 @@ sure: periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 3 - # Readiness uses "/" instead, because that path provably reaches the - # database. Read from the source of 0.7.4, not measured: every request runs - # ApplicationController's authenticate_user! before_action - # (app/controllers/concerns/authentication.rb); without a session cookie - - # which a kubelet probe never has - it falls into self_hosted_first_login?, - # which is "config.app_mode.self_hosted? && User.count.zero?". User.count is - # an SQL query. With the database up the request ends in a 302 to the login - # or registration page, which the kubelet counts as success (2xx-3xx); with - # the database down the query raises and Rails answers 500. + # Readiness asks for "/" instead, because that path provably reaches the + # database - and it does so through curl rather than an httpGet probe, for + # a reason that was measured rather than guessed (see below). # - # Two conditions this depends on, both of them chart-set: - # - SELF_HOSTED=true. In managed mode the same code short-circuits before - # User.count and "/" would redirect without touching Postgres. - # - the probe must not be answered by a redirect BEFORE the controller. - # Sure's production.rb defaults RAILS_FORCE_SSL to true, and a forced - # redirect to https is a 301 - i.e. a "success" that proves nothing. - # The chart sets RAILS_ASSUME_SSL alongside it (see the env block that - # the templates always render), so requests are treated as SSL and no - # redirect happens. Overriding RAILS_ASSUME_SSL to "false" while - # RAILS_FORCE_SSL stays "true" turns this probe back into a lie. + # Why "/" reaches the database: every request runs ApplicationController's + # authenticate_user! before_action (app/controllers/concerns/ + # authentication.rb); without a session cookie - which a probe never has - + # it falls into self_hosted_first_login?, i.e. + # "config.app_mode.self_hosted? && User.count.zero?". User.count is an SQL + # query. Measured in the install-test: against an empty database the app + # answered 302 to /registration/new, which is the branch that query decides, + # so the path demonstrably touches Postgres. With the database down the + # query raises and Rails answers 500. + # + # Why exec + curl and not httpGet: the kubelet's HTTP prober FOLLOWS + # redirects to the same host, and with RAILS_ASSUME_SSL=true (which the + # chart sets whenever the Ingress is on) Rails builds that redirect as + # https://:3000/... - a scheme Puma does not speak. The install-test + # failed with "http: server gave HTTP response to HTTPS client" on a pod + # that was answering correctly. curl without -L asks once and reports the + # status code; the test accepts 2xx and 3xx exactly as the kubelet would, + # and a 500 from a missing database still fails. curl is part of the image + # (the Dockerfile installs it), so nothing extra is pulled in for this. + # + # One condition this depends on: SELF_HOSTED=true, which the chart always + # sets. In managed mode the same code short-circuits before User.count and + # "/" would redirect without touching Postgres at all. readinessProbe: - httpGet: - path: / - port: http + exec: + command: + - /bin/bash + - -c + - "code=\"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:3000/)\"; test \"$code\" -ge 200 -a \"$code\" -lt 400" periodSeconds: 10 - timeoutSeconds: 5 + timeoutSeconds: 10 failureThreshold: 3 # The startup probe gates liveness and readiness, so neither needs an # initialDelaySeconds. Its budget has to cover the first start, which is the @@ -191,11 +212,14 @@ sure: # one operation that deterministically exceeds 512 MB (observed flat at # ~680 MB on a 2 GB container). The computed limit of 3 x 512Mi = 1536Mi # covers that; the request stays at the everyday figure. + # + # ephemeral-storage: same 256Mi as the web container and for the same + # measured reason - see the comment there. limitFactor: 3 requests: cpu: 0.1 memory: 512Mi - ephemeral-storage: 10Mi + ephemeral-storage: 256Mi # PROBES - the worker speaks no HTTP at all, so all three are exec probes. # # The command asks Sidekiq's own monitor whether THIS process is in the