{{- $settings := fromYaml (include "garage.settings" .) }} apiVersion: v1 kind: ConfigMap metadata: name: {{ .Release.Name }}-{{ .Chart.Name }}-configmap labels: app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs data: # The complete garage.toml. It carries NO SECRET: the RPC secret arrives as # the environment variable GARAGE_RPC_SECRET (see the StatefulSet and # values.yaml for why the file form cannot be used here), so this ConfigMap # is safe to read and the checksum/config annotation on the pod template # still means "the configuration changed". # # It is mounted with subPath at /etc/garage.toml, which is the path Garage # looks at without being told (-c defaults to /etc/garage.toml). A subPath # mount is never refreshed by the kubelet, which is exactly why the checksum # annotation is mandatory rather than decorative. garage.toml: | # Managed by the garage Helm chart - changes here are overwritten on the # next helm upgrade. Edit the chart's values instead. # ONE node, and the replication factor is 1 BY DECISION, with the risk # named (Gitea #81): longhorn-encrypted-ns already keeps three replicas of # the encrypted volume, and Garage replication 3 on top of that would be # nine times the raw data for two layers that know nothing about each # other. # # WHAT THAT COSTS, because it is not free: upstream says of # replication_factor = 1 "Do not use this for anything else than test # deployments", and the reason is on its known-issues page - the same # factor applies to the METADATA, so a single-replica setup is especially # exposed to metadata corruption, and a corrupted database file is # "generally not recoverable". Longhorn does not help against that: it # replicates a corrupted file three times. Redundancy against device # failure, yes; against file corruption, no. The three settings below are # the countermeasures that do work at factor 1, and none of them is # optional. replication_factor = {{ $settings.replicationFactor | int64 }} # LMDB is Garage's default since 0.9 and the faster engine. The price is # named: the reported corruption after an unclean shutdown stays possible, # the settings below only lower the probability, and at factor 1 there is # no peer to recover from - only the last snapshot. Upstream ties its LMDB # recommendation to replication factor >= 2; running it at 1 is this # chart's deliberate deviation. # # The alternative is "sqlite", which is slower on metadata throughput. With # a single mail server as the only consumer that ceiling is unlikely to be # reached, so sqlite is a defensible choice for anyone who weighs the # durability of a single-file database higher. "fjall" is experimental. db_engine = "{{ $settings.dbEngine }}" # metadata_dir and data_dir are two SUBDIRECTORIES OF ONE VOLUME; the # snapshots live on a second one. Longhorn snapshots are atomic PER VOLUME # and never across volumes, so keeping the index and the blocks together is # what prevents the one failure mode that matters for a mail store: a # backup whose metadata points at objects whose blocks were not written yet # ("the mailbox lists a message whose body is missing"). # # The snapshots are on the OTHER volume for the opposite reason: a recovery # point must not die with the volume whose corruption it is there to heal. metadata_dir = "{{ include "garage.metadataDir" . }}" data_dir = "{{ include "garage.dataDir" . }}" metadata_snapshots_dir = "{{ include "garage.snapshotsDir" . }}" # metadata_fsync IS OFF IN GARAGE'S DEFAULTS, and this is the single # setting most easily lost in a "tidy up the obvious defaults" pass. It is # set here on purpose: Longhorn's block layer cannot help if the engine # never calls fsync - a write that was never handed to the kernel is not in # any replica and not in any snapshot. Garage's own example configuration # pairs it with metadata_auto_snapshot_interval, as here. metadata_fsync = {{ $settings.metadataFsync }} # data_fsync stays off, which is Garage's default and the right trade for # the blocks: a lost block is one object, is detectable by a scrub, and is # replaceable from the sender's copy; a lost metadata write can take the # whole index with it. Turning it on costs an fsync per written block. data_fsync = {{ $settings.dataFsync }} # THE ONLY CONSISTENT RECOVERY POINT THIS DEPLOYMENT HAS. File-system # snapshots of metadata_dir guarantee nothing - Garage says so itself: a # snapshot taken during a metadata write is corrupt and useless as a # restore point, which is why Garage recommends ITS OWN snapshot mechanism # in all cases. This is the correction that reversed a premise of the # original plan: Longhorn backs up the blocks, not the metadata. # # KNOW THE WINDOW: Garage keeps exactly TWO snapshots (KEEP_SNAPSHOTS = 2), # the first one after interval/2 and every later one after interval times a # random 1.0-1.2. At "6h" that means the first after ~3h and a restore # window of roughly 6-14.4h afterwards. A corruption discovered later than # that is older than every snapshot that still exists. metadata_auto_snapshot_interval = "{{ $settings.metadataAutoSnapshotInterval }}" block_size = "{{ $settings.blockSize }}" # An integer is the zstd level; the literal "none" disables compression and # has to be QUOTED to stay valid TOML. Garage's deserializer takes exactly # those two forms - and 0 is not the second one: it is a valid integer that # zstd reads as its default level (3), so writing 0 to "switch compression # off" quietly turns it up. {{- if kindIs "string" $settings.compressionLevel }} compression_level = {{ $settings.compressionLevel | quote }} {{- else }} compression_level = {{ $settings.compressionLevel | int64 }} {{- end }} # Inter-node RPC. There are no other nodes here (no bootstrap_peers, no # kubernetes_discovery - see values.yaml), but Garage binds this listener # regardless and refuses to start without an rpc_secret, which arrives from # the environment. rpc_bind_addr = "{{ $settings.bindAddress }}:{{ $settings.ports.rpc | int64 }}" [s3_api] api_bind_addr = "{{ $settings.bindAddress }}:{{ $settings.ports.s3 | int64 }}" # The region name clients must use. It is not a cloud region, it is a label # both sides have to agree on - a consumer configured for a different # region gets a signature error, not a redirect. s3_region = "{{ $settings.s3.region }}" {{- with $settings.s3.rootDomain }} # Vhost-style bucket access (.). Only useful with an # Ingress and a wildcard DNS entry in front of it; path-style access works # without it and is what an in-cluster consumer uses. root_domain = {{ . | quote }} {{- end }} {{- if $settings.web.enabled }} [s3_web] # Static website hosting, OFF by default (see values.yaml): it is a second # HTTP listener and serves nothing at all without a root_domain, while the # consumer this chart was built for speaks S3. bind_addr = "{{ $settings.bindAddress }}:{{ $settings.ports.web | int64 }}" {{- with $settings.web.rootDomain }} root_domain = {{ . | quote }} {{- end }} {{- end }} [admin] # Bound on 0.0.0.0 rather than [::] BECAUSE ALL THREE PROBES USE THIS PORT. # A dual-stack listener on [::] only accepts the kubelet's IPv4 connection # while net.ipv6.bindv6only is 0 - true by default on Linux, but a node # sysctl this chart cannot see, and the failure would be a pod that never # becomes Ready for a reason nothing in the manifest points at. The S3 and # RPC listeners keep the upstream [::] default; set garage.bindAddress to # "0.0.0.0" on an IPv4-only cluster, or garage.adminBindAddress to "[::]" # on an IPv6-only one. # # NO admin_token IS SET, and that is not an omission: /health, the one # endpoint the probes use, is explicitly token-free (@special in the admin # router), and nothing in this deployment calls the authenticated API - the # CLI inside the pod talks RPC, not HTTP. A token here would be a fifth # vault field guarding nothing that is in use. # # WHAT IS AND IS NOT REACHABLE AS A RESULT, stated precisely because "no # token" reads like "nothing is open": # /health open, by design, and the probes depend on it # /metrics OPEN TOO - metrics_require_token defaults to false, so with # no metrics_token the endpoint needs no authentication. That # is internal metrics on a ClusterIP port; set # metrics_require_token = true plus a token to close it. # everything else 401/403 until a token exists. Note that a config # token is not the only way to get one: Garage 2.x also keeps # admin tokens in the cluster ("garage admin-token create"), # and those work without any entry in this file. # If you do want a config token, it is admin_token_file / the # GARAGE_ADMIN_TOKEN environment variable - and the file form carries the # same permission trap as rpc_secret_file (see values.yaml). api_bind_addr = "{{ $settings.adminBindAddress }}:{{ $settings.ports.admin | int64 }}"