jklein
a414af089d
The first install-test run failed with both containers crash-looping during boot: system temporary path is world-writable: /tmp /tmp is world-writable: /tmp . is not writable: /rails ArgumentError: could not find a temporary directory raised from MiniMagick, which Active Storage loads while Rails boots. Ruby's Dir.tmpdir skips a candidate that is world-writable without a sticky bit, and a Kubernetes emptyDir is created mode 0777 with no sticky bit - so with a read-only root filesystem there was no candidate left. A tiny init container, the app's own image running as the same non-root uid and with no capability added, now creates one private subdirectory (mode 0700) inside each tmp emptyDir, and both containers run with TMPDIR pointing at it. Creating an entry in a 0777 directory and chmod'ing what one just created needs no privilege, so the hardening standard holds unchanged.
230 lines
12 KiB
YAML
230 lines
12 KiB
YAML
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
labels:
|
|
app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
spec:
|
|
serviceName: {{ .Release.Name }}-{{ .Chart.Name }}-service
|
|
replicas: {{ include "sure.replicas" (dict "scaleDown" .Values.scaleDown "replicas" .Values.sure.replicas) }}
|
|
selector:
|
|
matchLabels:
|
|
app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
template:
|
|
metadata:
|
|
name: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
labels:
|
|
app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
|
|
annotations:
|
|
# The chart has no ConfigMap, so there is no checksum/config here. What
|
|
# these annotations cover is the SOURCE of the secrets, not their
|
|
# content: the Kubernetes Secrets are materialized by the 1Password
|
|
# operator, so Helm never sees what is in them. Repointing one of the
|
|
# secrets.*SecretRef values at a different item therefore changes the
|
|
# pod template and rolls the pod, instead of silently leaving the old
|
|
# credentials in the running container. A change of an item's CONTENT
|
|
# inside the vault is not covered - see the comment on secrets in
|
|
# values.yaml for why that gap cannot be closed here.
|
|
# The name differs from checksum/config on purpose - that one promises
|
|
# "the config changed", this one only "the config source changed".
|
|
checksum/secret-ref: {{ include (print $.Template.BasePath "/sure.secret.yaml") . | sha256sum }}
|
|
checksum/secret-ref-db: {{ include (print $.Template.BasePath "/sure-db.secret.yaml") . | sha256sum }}
|
|
checksum/secret-ref-redis: {{ include (print $.Template.BasePath "/sure-redis.secret.yaml") . | sha256sum }}
|
|
spec:
|
|
automountServiceAccountToken: false
|
|
securityContext:
|
|
{{- include "sure.podSecurityContext" .Values.sure.securityContext | nindent 8 }}
|
|
initContainers:
|
|
# Ruby refuses a world-writable directory that has no sticky bit as its
|
|
# temporary directory (ruby 3.4, tmpdir.rb: a candidate is skipped when
|
|
# "world_writable? && !sticky?"), and a Kubernetes emptyDir is created
|
|
# mode 0777 WITHOUT the sticky bit. With /tmp mounted as an emptyDir and
|
|
# the image's root filesystem read-only, Ruby then finds no candidate at
|
|
# all and both containers die during boot - measured in this chart's
|
|
# first install-test, which logged
|
|
#
|
|
# system temporary path is world-writable: /tmp
|
|
# /tmp is world-writable: /tmp
|
|
# . is not writable: /rails
|
|
# ArgumentError: could not find a temporary directory
|
|
#
|
|
# raised from MiniMagick, which Active Storage loads while Rails boots.
|
|
# Note where it does NOT come from: nothing in the app asked for a temp
|
|
# file yet.
|
|
#
|
|
# The fix is a directory that is writable without being world-writable.
|
|
# This init container creates one per app container inside the same
|
|
# emptyDirs (mode 0700, owned by the pod's uid) and both containers get
|
|
# TMPDIR pointed at it. It runs as the same non-root uid as the app and
|
|
# needs no capability at all: the emptyDir root is 0777, so creating an
|
|
# entry in it is allowed, and chmod on a directory one has just created
|
|
# needs no privilege. That is deliberately weaker than the chown-as-root
|
|
# init container paperless-ngx needs.
|
|
#
|
|
# The image is the app's own, so there is no second image to pin, no
|
|
# Renovate block to maintain for it and nothing extra to pull.
|
|
- name: {{ .Release.Name }}-{{ .Chart.Name }}-init-tmp
|
|
image: "ghcr.io/we-promise/sure:{{ .Chart.AppVersion }}"
|
|
imagePullPolicy: IfNotPresent
|
|
# command, not args: the image entrypoint would start the application.
|
|
command:
|
|
- /bin/bash
|
|
- -ec
|
|
- |
|
|
for dir in /web-tmp/ruby /worker-tmp/ruby; do
|
|
mkdir -p "$dir"
|
|
chmod 700 "$dir"
|
|
done
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
runAsNonRoot: true
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
volumeMounts:
|
|
- name: web-tmp
|
|
mountPath: /web-tmp
|
|
- name: worker-tmp
|
|
mountPath: /worker-tmp
|
|
# Fixed, deliberately small: this container runs two mkdirs and two
|
|
# chmods. Not values-configurable, for the same reason as the init
|
|
# container in paperless-ngx - there is nothing here to tune.
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
ephemeral-storage: 1Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
ephemeral-storage: 10Mi
|
|
containers:
|
|
# The web process. Its command is the image's default
|
|
# (CMD ["./bin/rails", "server"]), which is exactly what makes the
|
|
# entrypoint run "./bin/rails db:prepare" first - the migration only
|
|
# happens for that command, so it happens here and nowhere else.
|
|
- name: {{ .Release.Name }}-{{ .Chart.Name }}-web
|
|
image: "ghcr.io/we-promise/sure:{{ .Chart.AppVersion }}"
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
{{- include "sure.commonEnv" . | nindent 12 }}
|
|
{{- include "sure.env" (dict "root" $ "env" .Values.sure.web.env) | nindent 12 }}
|
|
ports:
|
|
- containerPort: 3000
|
|
protocol: TCP
|
|
name: http
|
|
# Probes, resources and both security contexts go through the
|
|
# null-safe helpers (#99): an erased block means "chart defaults
|
|
# apply" instead of a vanished probe or an unhardened container.
|
|
# Never wrap these in an "if" - a guard hides the erased case
|
|
# completely instead of rendering a visible null.
|
|
livenessProbe:
|
|
{{- include "sure.webLivenessProbe" .Values.sure.web.livenessProbe | nindent 12 }}
|
|
readinessProbe:
|
|
{{- include "sure.webReadinessProbe" .Values.sure.web.readinessProbe | nindent 12 }}
|
|
startupProbe:
|
|
{{- include "sure.webStartupProbe" .Values.sure.web.startupProbe | nindent 12 }}
|
|
resources:
|
|
{{- include "sure.webResources" .Values.sure.web.resources | nindent 12 }}
|
|
securityContext:
|
|
{{- include "sure.containerSecurityContext" (dict "block" .Values.sure.web.securityContext "path" "sure.web.securityContext") | nindent 12 }}
|
|
volumeMounts:
|
|
# Active Storage's local disk service, i.e. every uploaded file and
|
|
# import, lives here (config/storage.yml: Rails.root.join("storage")).
|
|
- name: storage
|
|
mountPath: /rails/storage
|
|
# Writable paths, so readOnlyRootFilesystem: true can stay on.
|
|
# Each container gets its own emptyDirs: nothing has to be shared
|
|
# between the two processes, and a shared scratch directory would
|
|
# only create coupling nobody asked for.
|
|
#
|
|
# /tmp: Rack writes multipart uploads (CSV imports, attachments)
|
|
# here before Active Storage ever sees them. The app does not use
|
|
# this path directly, though - TMPDIR points at the private
|
|
# subdirectory the init container creates inside it, because Ruby
|
|
# rejects a world-writable emptyDir as a temporary directory.
|
|
- name: web-tmp
|
|
mountPath: /tmp
|
|
# /rails/tmp: Rails' own scratch directory - the Bootsnap cache,
|
|
# Puma's pidfile when PIDFILE is set, and the combined CA bundle the
|
|
# 00_ssl initializer writes when SSL_CA_FILE is given. Note the
|
|
# trade-off, read off the Dockerfile: the image ships a Bootsnap
|
|
# cache precompiled at build time under this path, and mounting an
|
|
# emptyDir over it hides that cache, so every pod start recompiles
|
|
# it. Boot time cost, not correctness; the alternative is a
|
|
# writable image layer, which the hardening standard rules out.
|
|
- name: web-rails-tmp
|
|
mountPath: /rails/tmp
|
|
# /rails/log: production logs go to STDOUT (config/environments/
|
|
# production.rb), so under normal operation nothing is written here
|
|
# at all. It stays writable because Rails' default logger path is
|
|
# log/<env>.log and any rake or console path that falls back to it
|
|
# would otherwise hit a read-only filesystem. Not measured.
|
|
- name: web-rails-log
|
|
mountPath: /rails/log
|
|
# The Sidekiq process. Same image, different command - and because that
|
|
# command is NOT "./bin/rails server", the image entrypoint skips
|
|
# db:prepare here, so the two containers never migrate against each
|
|
# other.
|
|
#
|
|
# args, NOT command, and that distinction is load-bearing: Kubernetes'
|
|
# "command" replaces the image's ENTRYPOINT, which here is
|
|
# /rails/bin/docker-entrypoint - the script that preloads jemalloc.
|
|
# Upstream counts jemalloc among the three things that make the
|
|
# documented memory figures hold, so replacing the entrypoint would
|
|
# silently give this container a different memory profile than the one
|
|
# its requests are sized for. "args" replaces only the CMD, which is
|
|
# what upstream's compose file does too.
|
|
- name: {{ .Release.Name }}-{{ .Chart.Name }}-worker
|
|
image: "ghcr.io/we-promise/sure:{{ .Chart.AppVersion }}"
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- bundle
|
|
- exec
|
|
- sidekiq
|
|
env:
|
|
{{- include "sure.commonEnv" . | nindent 12 }}
|
|
{{- include "sure.env" (dict "root" $ "env" .Values.sure.worker.env) | nindent 12 }}
|
|
livenessProbe:
|
|
{{- include "sure.workerLivenessProbe" .Values.sure.worker.livenessProbe | nindent 12 }}
|
|
readinessProbe:
|
|
{{- include "sure.workerReadinessProbe" .Values.sure.worker.readinessProbe | nindent 12 }}
|
|
startupProbe:
|
|
{{- include "sure.workerStartupProbe" .Values.sure.worker.startupProbe | nindent 12 }}
|
|
resources:
|
|
{{- include "sure.workerResources" .Values.sure.worker.resources | nindent 12 }}
|
|
securityContext:
|
|
{{- include "sure.containerSecurityContext" (dict "block" .Values.sure.worker.securityContext "path" "sure.worker.securityContext") | nindent 12 }}
|
|
volumeMounts:
|
|
# The worker reads and writes the same Active Storage files as the
|
|
# web process (imports are processed as background jobs), which is
|
|
# why upstream's compose file mounts one volume into both and why
|
|
# both containers live in one pod here: a single ReadWriteOnce PVC
|
|
# cannot be attached to two pods.
|
|
- name: storage
|
|
mountPath: /rails/storage
|
|
- name: worker-tmp
|
|
mountPath: /tmp
|
|
- name: worker-rails-tmp
|
|
mountPath: /rails/tmp
|
|
- name: worker-rails-log
|
|
mountPath: /rails/log
|
|
restartPolicy: Always
|
|
volumes:
|
|
- name: storage
|
|
persistentVolumeClaim:
|
|
claimName: {{ .Release.Name }}-{{ .Chart.Name }}-pvc
|
|
- name: web-tmp
|
|
emptyDir: {}
|
|
- name: web-rails-tmp
|
|
emptyDir: {}
|
|
- name: web-rails-log
|
|
emptyDir: {}
|
|
- name: worker-tmp
|
|
emptyDir: {}
|
|
- name: worker-rails-tmp
|
|
emptyDir: {}
|
|
- name: worker-rails-log
|
|
emptyDir: {}
|