jklein
d9267b2180
Second install-test run, three findings, all measured: - Readiness on "/" failed with "http: server gave HTTP response to HTTPS client" on a pod that was answering correctly. The kubelet's HTTP prober follows redirects to the same host, and with RAILS_ASSUME_SSL=true the app builds its redirect as https://<pod-ip>:3000/registration/new, which Puma does not speak. The probe now runs curl without -L and accepts 2xx-3xx itself. The same log also proves what the probe is for: against an empty database the redirect target was /registration/new, the branch User.count decides, so the path really does reach Postgres. - The pod was evicted within two minutes of every start: "ephemeral local storage usage exceeds the total limit of containers 60Mi". emptyDirs and the stdout log both count there, and the image rebuilds its Bootsnap cache under /rails/tmp on every start because the emptyDir hides the one baked into the image. The request is now 256Mi (limit 768Mi) per app container - headroom, not a measurement. - /home/rails is read-only, so Bundler created a throwaway home under TMPDIR on every invocation, worker probe included. HOME now points at a directory the init container creates. The CI postgres fixture gets -d sure on its readiness probe, so the server stops logging a missing database that nothing asked for.
232 lines
12 KiB
YAML
232 lines
12 KiB
YAML
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
labels:
|
|
app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
spec:
|
|
serviceName: {{ .Release.Name }}-{{ .Chart.Name }}-service
|
|
replicas: {{ include "sure.replicas" (dict "scaleDown" .Values.scaleDown "replicas" .Values.sure.replicas) }}
|
|
selector:
|
|
matchLabels:
|
|
app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
template:
|
|
metadata:
|
|
name: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
labels:
|
|
app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
annotations:
|
|
# The chart has no ConfigMap, so there is no checksum/config here. What
|
|
# these annotations cover is the SOURCE of the secrets, not their
|
|
# content: the Kubernetes Secrets are materialized by the 1Password
|
|
# operator, so Helm never sees what is in them. Repointing one of the
|
|
# secrets.*SecretRef values at a different item therefore changes the
|
|
# pod template and rolls the pod, instead of silently leaving the old
|
|
# credentials in the running container. A change of an item's CONTENT
|
|
# inside the vault is not covered - see the comment on secrets in
|
|
# values.yaml for why that gap cannot be closed here.
|
|
# The name differs from checksum/config on purpose - that one promises
|
|
# "the config changed", this one only "the config source changed".
|
|
checksum/secret-ref: {{ include (print $.Template.BasePath "/sure.secret.yaml") . | sha256sum }}
|
|
checksum/secret-ref-db: {{ include (print $.Template.BasePath "/sure-db.secret.yaml") . | sha256sum }}
|
|
checksum/secret-ref-redis: {{ include (print $.Template.BasePath "/sure-redis.secret.yaml") . | sha256sum }}
|
|
spec:
|
|
automountServiceAccountToken: false
|
|
securityContext:
|
|
{{- include "sure.podSecurityContext" .Values.sure.securityContext | nindent 8 }}
|
|
initContainers:
|
|
# Ruby refuses a world-writable directory that has no sticky bit as its
|
|
# temporary directory (ruby 3.4, tmpdir.rb: a candidate is skipped when
|
|
# "world_writable? && !sticky?"), and a Kubernetes emptyDir is created
|
|
# mode 0777 WITHOUT the sticky bit. With /tmp mounted as an emptyDir and
|
|
# the image's root filesystem read-only, Ruby then finds no candidate at
|
|
# all and both containers die during boot - measured in this chart's
|
|
# first install-test, which logged
|
|
#
|
|
# system temporary path is world-writable: /tmp
|
|
# /tmp is world-writable: /tmp
|
|
# . is not writable: /rails
|
|
# ArgumentError: could not find a temporary directory
|
|
#
|
|
# raised from MiniMagick, which Active Storage loads while Rails boots.
|
|
# Note where it does NOT come from: nothing in the app asked for a temp
|
|
# file yet.
|
|
#
|
|
# The fix is a directory that is writable without being world-writable.
|
|
# This init container creates two of them per app container inside the
|
|
# same emptyDirs (mode 0700, owned by the pod's uid): "ruby" for TMPDIR,
|
|
# and "home" for HOME - the image's /home/rails is read-only, and
|
|
# Bundler answers that by creating a throwaway home under TMPDIR on
|
|
# every single invocation. It runs as the same non-root uid as the app
|
|
# and needs no capability at all: the emptyDir root is 0777, so creating an
|
|
# entry in it is allowed, and chmod on a directory one has just created
|
|
# needs no privilege. That is deliberately weaker than the chown-as-root
|
|
# init container paperless-ngx needs.
|
|
#
|
|
# The image is the app's own, so there is no second image to pin, no
|
|
# Renovate block to maintain for it and nothing extra to pull.
|
|
- name: {{ .Release.Name }}-{{ .Chart.Name }}-init-tmp
|
|
image: "ghcr.io/we-promise/sure:{{ .Chart.AppVersion }}"
|
|
imagePullPolicy: IfNotPresent
|
|
# command, not args: the image entrypoint would start the application.
|
|
command:
|
|
- /bin/bash
|
|
- -ec
|
|
- |
|
|
for dir in /web-tmp/ruby /web-tmp/home /worker-tmp/ruby /worker-tmp/home; do
|
|
mkdir -p "$dir"
|
|
chmod 700 "$dir"
|
|
done
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
volumeMounts:
|
|
- name: web-tmp
|
|
mountPath: /web-tmp
|
|
- name: worker-tmp
|
|
mountPath: /worker-tmp
|
|
# Fixed, deliberately small: this container runs two mkdirs and two
|
|
# chmods. Not values-configurable, for the same reason as the init
|
|
# container in paperless-ngx - there is nothing here to tune.
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
ephemeral-storage: 1Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
ephemeral-storage: 10Mi
|
|
containers:
|
|
# The web process. Its command is the image's default
|
|
# (CMD ["./bin/rails", "server"]), which is exactly what makes the
|
|
# entrypoint run "./bin/rails db:prepare" first - the migration only
|
|
# happens for that command, so it happens here and nowhere else.
|
|
- name: {{ .Release.Name }}-{{ .Chart.Name }}-web
|
|
image: "ghcr.io/we-promise/sure:{{ .Chart.AppVersion }}"
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
{{- include "sure.commonEnv" . | nindent 12 }}
|
|
{{- include "sure.env" (dict "root" $ "env" .Values.sure.web.env) | nindent 12 }}
|
|
ports:
|
|
- containerPort: 3000
|
|
protocol: TCP
|
|
name: http
|
|
# Probes, resources and both security contexts go through the
|
|
# null-safe helpers (#99): an erased block means "chart defaults
|
|
# apply" instead of a vanished probe or an unhardened container.
|
|
# Never wrap these in an "if" - a guard hides the erased case
|
|
# completely instead of rendering a visible null.
|
|
livenessProbe:
|
|
{{- include "sure.webLivenessProbe" .Values.sure.web.livenessProbe | nindent 12 }}
|
|
readinessProbe:
|
|
{{- include "sure.webReadinessProbe" .Values.sure.web.readinessProbe | nindent 12 }}
|
|
startupProbe:
|
|
{{- include "sure.webStartupProbe" .Values.sure.web.startupProbe | nindent 12 }}
|
|
resources:
|
|
{{- include "sure.webResources" .Values.sure.web.resources | nindent 12 }}
|
|
securityContext:
|
|
{{- include "sure.containerSecurityContext" (dict "block" .Values.sure.web.securityContext "path" "sure.web.securityContext") | nindent 12 }}
|
|
volumeMounts:
|
|
# Active Storage's local disk service, i.e. every uploaded file and
|
|
# import, lives here (config/storage.yml: Rails.root.join("storage")).
|
|
- name: storage
|
|
mountPath: /rails/storage
|
|
# Writable paths, so readOnlyRootFilesystem: true can stay on.
|
|
# Each container gets its own emptyDirs: nothing has to be shared
|
|
# between the two processes, and a shared scratch directory would
|
|
# only create coupling nobody asked for.
|
|
#
|
|
# /tmp: Rack writes multipart uploads (CSV imports, attachments)
|
|
# here before Active Storage ever sees them. The app does not use
|
|
# this path directly, though - TMPDIR points at the private
|
|
# subdirectory the init container creates inside it, because Ruby
|
|
# rejects a world-writable emptyDir as a temporary directory.
|
|
- name: web-tmp
|
|
mountPath: /tmp
|
|
# /rails/tmp: Rails' own scratch directory - the Bootsnap cache,
|
|
# Puma's pidfile when PIDFILE is set, and the combined CA bundle the
|
|
# 00_ssl initializer writes when SSL_CA_FILE is given. Note the
|
|
# trade-off, read off the Dockerfile: the image ships a Bootsnap
|
|
# cache precompiled at build time under this path, and mounting an
|
|
# emptyDir over it hides that cache, so every pod start recompiles
|
|
# it. Boot time cost, not correctness; the alternative is a
|
|
# writable image layer, which the hardening standard rules out.
|
|
- name: web-rails-tmp
|
|
mountPath: /rails/tmp
|
|
# /rails/log: production logs go to STDOUT (config/environments/
|
|
# production.rb), so under normal operation nothing is written here
|
|
# at all. It stays writable because Rails' default logger path is
|
|
# log/<env>.log and any rake or console path that falls back to it
|
|
# would otherwise hit a read-only filesystem. Not measured.
|
|
- name: web-rails-log
|
|
mountPath: /rails/log
|
|
# The Sidekiq process. Same image, different command - and because that
|
|
# command is NOT "./bin/rails server", the image entrypoint skips
|
|
# db:prepare here, so the two containers never migrate against each
|
|
# other.
|
|
#
|
|
# args, NOT command, and that distinction is load-bearing: Kubernetes'
|
|
# "command" replaces the image's ENTRYPOINT, which here is
|
|
# /rails/bin/docker-entrypoint - the script that preloads jemalloc.
|
|
# Upstream counts jemalloc among the three things that make the
|
|
# documented memory figures hold, so replacing the entrypoint would
|
|
# silently give this container a different memory profile than the one
|
|
# its requests are sized for. "args" replaces only the CMD, which is
|
|
# what upstream's compose file does too.
|
|
- name: {{ .Release.Name }}-{{ .Chart.Name }}-worker
|
|
image: "ghcr.io/we-promise/sure:{{ .Chart.AppVersion }}"
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- bundle
|
|
- exec
|
|
- sidekiq
|
|
env:
|
|
{{- include "sure.commonEnv" . | nindent 12 }}
|
|
{{- include "sure.env" (dict "root" $ "env" .Values.sure.worker.env) | nindent 12 }}
|
|
livenessProbe:
|
|
{{- include "sure.workerLivenessProbe" .Values.sure.worker.livenessProbe | nindent 12 }}
|
|
readinessProbe:
|
|
{{- include "sure.workerReadinessProbe" .Values.sure.worker.readinessProbe | nindent 12 }}
|
|
startupProbe:
|
|
{{- include "sure.workerStartupProbe" .Values.sure.worker.startupProbe | nindent 12 }}
|
|
resources:
|
|
{{- include "sure.workerResources" .Values.sure.worker.resources | nindent 12 }}
|
|
securityContext:
|
|
{{- include "sure.containerSecurityContext" (dict "block" .Values.sure.worker.securityContext "path" "sure.worker.securityContext") | nindent 12 }}
|
|
volumeMounts:
|
|
# The worker reads and writes the same Active Storage files as the
|
|
# web process (imports are processed as background jobs), which is
|
|
# why upstream's compose file mounts one volume into both and why
|
|
# both containers live in one pod here: a single ReadWriteOnce PVC
|
|
# cannot be attached to two pods.
|
|
- name: storage
|
|
mountPath: /rails/storage
|
|
- name: worker-tmp
|
|
mountPath: /tmp
|
|
- name: worker-rails-tmp
|
|
mountPath: /rails/tmp
|
|
- name: worker-rails-log
|
|
mountPath: /rails/log
|
|
restartPolicy: Always
|
|
volumes:
|
|
- name: storage
|
|
persistentVolumeClaim:
|
|
claimName: {{ .Release.Name }}-{{ .Chart.Name }}-pvc
|
|
- name: web-tmp
|
|
emptyDir: {}
|
|
- name: web-rails-tmp
|
|
emptyDir: {}
|
|
- name: web-rails-log
|
|
emptyDir: {}
|
|
- name: worker-tmp
|
|
emptyDir: {}
|
|
- name: worker-rails-tmp
|
|
emptyDir: {}
|
|
- name: worker-rails-log
|
|
emptyDir: {}
|