{{- $settings := fromYaml (include "garage.settings" .) }} {{- /* Catch the one flag combination Garage rejects at START, at render time instead: --default-access-key and --default-bucket "can only be used in single-node mode" (src/garage/server.rs). Without this check the chart would install cleanly and the container would exit immediately, with the reason only in the pod log. */ -}} {{- if and (not $settings.singleNode) (or $settings.defaultAccessKey $settings.defaultBucket) }} {{- fail "garage.defaultAccessKey and garage.defaultBucket require garage.singleNode: true - Garage refuses to start otherwise (\"Flags --default-access-key and --default-bucket can only be used in single-node mode\"). With singleNode: false, create the key and the bucket by hand after applying the cluster layout; NOTES.txt has the commands." }} {{- end }} {{- $importKey := and $settings.singleNode (or $settings.defaultAccessKey $settings.defaultBucket) }} apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ .Release.Name }}-{{ .Chart.Name }}-sfs labels: app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs spec: replicas: {{ include "garage.replicas" (dict "scaleDown" .Values.scaleDown "replicas" .Values.garage.replicas) }} serviceName: {{ .Release.Name }}-{{ .Chart.Name }}-service selector: matchLabels: app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs template: metadata: name: {{ .Release.Name }}-{{ .Chart.Name }}-sfs labels: app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs annotations: # Hash of the rendered ConfigMap: a changed garage.toml changes the pod # template and thus restarts the pod. Without it the change would reach # nothing - the config is mounted with subPath (never refreshed by the # kubelet) and Garage reads it once, at start. checksum/config: {{ include (print $.Template.BasePath "/garage.configmap.yaml") . | sha256sum }} # Hash of the rendered OnePasswordItems. This covers the SOURCE of the # secrets, not their content: repointing secrets.rpcSecretRef or # secrets.s3SecretRef at a different vault item rolls the pod instead of # leaving the old credentials in the running process. A change INSIDE an # item cannot be covered - Helm never sees it (see garage.secret.yaml). checksum/secret-ref: {{ include (print $.Template.BasePath "/garage.secret.yaml") . | sha256sum }} spec: automountServiceAccountToken: false securityContext: {{- include "garage.podSecurityContext" .Values.garage.securityContext | nindent 8 }} # NO INIT CONTAINER, and that is a property of the image rather than a # choice: dxflrs/garage is built FROM scratch and contains exactly one # file, /garage. There is no shell, no mkdir, no wget - so an init # container would have to be a SECOND image to pin, track and update. # Nothing needs one: Garage creates metadata_dir, data_dir and the # snapshots directory itself on start (create_dir_all in # src/model/garage.rs and src/model/snapshot.rs), and the volumes are made # writable by fsGroup. # # The same property rules out exec probes in this chart for good - all # three probes are HTTP or TCP, see the helpers. containers: - name: {{ .Release.Name }}-{{ .Chart.Name }}-sfs # Fully qualified: an unqualified "dxflrs/garage" resolves to Docker # Hub by convention rather than by rule. The multi-arch image, not # upstream's own chart's dxflrs/amd64_garage. image: "docker.io/dxflrs/garage:{{ .Chart.AppVersion }}" imagePullPolicy: IfNotPresent # command AND args, because the image's CMD is ["/garage", "server"] # and flags have to go after the subcommand. Overriding only args # would drop the "server" subcommand. command: - /garage args: - server {{- if $settings.singleNode }} # --single-node is what makes this chart's default install reach # Ready without an operator (Gitea #81, second plan correction): # initial_config() in src/garage/server.rs creates the cluster # layout in-process BEFORE the S3, web and admin servers are # spawned - zone "dc1", capacity taken from the detected disk size, # then apply_staged_changes(1). # # IT IS IDEMPOTENT, and it cannot damage an existing cluster. The # layout version decides: 0 creates the layout, 1 does nothing at # all, anything above 1 refuses to start ("Refusing to run in # single-node mode: layout version is already superior to 1"), and # a node that sees other nodes refuses as well ("more nodes are # already present in the cluster"). There is no --version argument # that could be passed twice with the same value, which is the trap # the CLI path (garage layout apply --version N+1) has. # # It requires replication_factor = 1 and says so loudly if it is # not. - --single-node {{- if or $settings.defaultAccessKey $settings.defaultBucket }} # Imports the access key from GARAGE_DEFAULT_ACCESS_KEY / # GARAGE_DEFAULT_SECRET_KEY below, i.e. FROM the vault item INTO # Garage. Re-running is guarded upstream: a key that already exists # with a different secret aborts the start, and one that was deleted # in the cluster cannot be added back. - --default-access-key {{- end }} {{- if $settings.defaultBucket }} # Creates the bucket from GARAGE_DEFAULT_BUCKET and grants the key # above read/write/owner on it. Implies --default-access-key. - --default-bucket {{- end }} {{- end }} env: # THE RPC SECRET AS AN ENVIRONMENT VARIABLE, NOT AS A FILE - and the # full reason is in values.yaml under secrets.rpcSecretRef. The # short version, because this is the line a later round will want to # "fix": rpc_secret_file CANNOT WORK in a non-root pod. # read_secret_file rejects any file whose mode has a group or other # bit set (src/garage/secrets.rs: mode & 0o077), and the kubelet # OR-s the group-read bit onto every file of a secret volume as soon # as fsGroup is set. Without fsGroup the file stays root-owned and # uid 20000 cannot read it. Kubernetes can set the group of such a # file, never its owner, so no combination satisfies both. - name: GARAGE_RPC_SECRET valueFrom: secretKeyRef: name: {{ .Release.Name }}-{{ .Chart.Name }}-rpc-secret key: rpc-secret {{- if $importKey }} - name: GARAGE_DEFAULT_ACCESS_KEY valueFrom: secretKeyRef: name: {{ .Release.Name }}-{{ .Chart.Name }}-s3-secret key: access-key - name: GARAGE_DEFAULT_SECRET_KEY valueFrom: secretKeyRef: name: {{ .Release.Name }}-{{ .Chart.Name }}-s3-secret key: secret-key {{- end }} {{- if and $settings.singleNode $settings.defaultBucket }} - name: GARAGE_DEFAULT_BUCKET valueFrom: secretKeyRef: name: {{ .Release.Name }}-{{ .Chart.Name }}-s3-secret key: bucket {{- end }} {{- with .Values.garage.env }} {{- include "garage.env" (dict "root" $ "env" .) | nindent 12 }} {{- end }} ports: - containerPort: {{ $settings.ports.s3 | int64 }} protocol: TCP name: s3 - containerPort: {{ $settings.ports.rpc | int64 }} protocol: TCP name: rpc {{- if $settings.web.enabled }} - containerPort: {{ $settings.ports.web | int64 }} protocol: TCP name: web {{- end }} # The port all three probes use. Its NAME is what the probe defaults # reference, so renaming it here breaks them. - containerPort: {{ $settings.ports.admin | int64 }} protocol: TCP name: admin # Probes, resources and both security contexts go through the # null-safe helpers (#99): an erased block means "chart defaults # apply" instead of a vanished probe or an unhardened container. # Never wrap these in an "if" - a guard hides the erased case # completely instead of rendering a visible null. # # Which probe points at what, and why liveness and startup must NOT # point at /health, is argued at each helper in _helpers.tpl. livenessProbe: {{- include "garage.livenessProbe" .Values.garage.livenessProbe | nindent 12 }} readinessProbe: {{- include "garage.readinessProbe" .Values.garage.readinessProbe | nindent 12 }} startupProbe: {{- include "garage.startupProbe" .Values.garage.startupProbe | nindent 12 }} resources: {{- include "garage.effectiveResources" .Values.garage.resources | nindent 12 }} securityContext: {{- include "garage.containerSecurityContext" .Values.garage.securityContext | nindent 12 }} volumeMounts: # ONE volume carrying metadata_dir and data_dir as subdirectories, # so a Longhorn snapshot catches index and blocks in one atomic # operation. Garage creates both subdirectories on start. - name: data mountPath: {{ include "garage.dataVolumePath" . }} # The SECOND volume, holding the only consistent recovery point this # deployment has. Separate on purpose: a snapshot that dies with the # volume whose corruption it should repair is not a recovery point. - name: snapshots mountPath: {{ include "garage.snapshotsDir" . }} - name: config mountPath: /etc/garage.toml subPath: garage.toml readOnly: true # No emptyDir for /tmp, and that is checked rather than assumed: with # readOnlyRootFilesystem: true the root filesystem of this container # holds exactly one file. Garage writes under metadata_dir, data_dir # and the snapshots directory and uses no temporary directory # (verified in src/model, src/db and src/block at v2.4.1). restartPolicy: Always volumes: - name: data persistentVolumeClaim: claimName: {{ .Release.Name }}-{{ .Chart.Name }}-pvc - name: snapshots persistentVolumeClaim: claimName: {{ .Release.Name }}-{{ .Chart.Name }}-snapshots-pvc - name: config configMap: name: {{ .Release.Name }}-{{ .Chart.Name }}-configmap