helm-charts/garage/templates/garage.configmap.yaml

{{- $settings := fromYaml (include "garage.settings" .) }}
apiVersion: v1
kind: ConfigMap
metadata:
  name: {{ .Release.Name }}-{{ .Chart.Name }}-configmap
  labels:
    app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs
data:
  # The complete garage.toml. It carries NO SECRET: the RPC secret arrives as
  # the environment variable GARAGE_RPC_SECRET (see the StatefulSet and
  # values.yaml for why the file form cannot be used here), so this ConfigMap
  # is safe to read and the checksum/config annotation on the pod template
  # still means "the configuration changed".
  #
  # It is mounted with subPath at /etc/garage.toml, which is the path Garage
  # looks at without being told (-c defaults to /etc/garage.toml). A subPath
  # mount is never refreshed by the kubelet, which is exactly why the checksum
  # annotation is mandatory rather than decorative.
  garage.toml: |
    # Managed by the garage Helm chart - changes here are overwritten on the
    # next helm upgrade. Edit the chart's values instead.

    # ONE node, and the replication factor is 1 BY DECISION, with the risk
    # named (Gitea #81): longhorn-encrypted-ns already keeps three replicas of
    # the encrypted volume, and Garage replication 3 on top of that would be
    # nine times the raw data for two layers that know nothing about each
    # other.
    #
    # WHAT THAT COSTS, because it is not free: upstream says of
    # replication_factor = 1 "Do not use this for anything else than test
    # deployments", and the reason is on its known-issues page - the same
    # factor applies to the METADATA, so a single-replica setup is especially
    # exposed to metadata corruption, and a corrupted database file is
    # "generally not recoverable". Longhorn does not help against that: it
    # replicates a corrupted file three times. Redundancy against device
    # failure, yes; against file corruption, no. The three settings below are
    # the countermeasures that do work at factor 1, and none of them is
    # optional.
    replication_factor = {{ $settings.replicationFactor | int64 }}

    # LMDB is Garage's default since 0.9 and the faster engine. The price is
    # named: the reported corruption after an unclean shutdown stays possible,
    # the settings below only lower the probability, and at factor 1 there is
    # no peer to recover from - only the last snapshot. Upstream ties its LMDB
    # recommendation to replication factor >= 2; running it at 1 is this
    # chart's deliberate deviation.
    #
    # The alternative is "sqlite", which is slower on metadata throughput. With
    # a single mail server as the only consumer that ceiling is unlikely to be
    # reached, so sqlite is a defensible choice for anyone who weighs the
    # durability of a single-file database higher. "fjall" is experimental.
    db_engine = "{{ $settings.dbEngine }}"

    # metadata_dir and data_dir are two SUBDIRECTORIES OF ONE VOLUME; the
    # snapshots live on a second one. Longhorn snapshots are atomic PER VOLUME
    # and never across volumes, so keeping the index and the blocks together is
    # what prevents the one failure mode that matters for a mail store: a
    # backup whose metadata points at objects whose blocks were not written yet
    # ("the mailbox lists a message whose body is missing").
    #
    # The snapshots are on the OTHER volume for the opposite reason: a recovery
    # point must not die with the volume whose corruption it is there to heal.
    metadata_dir = "{{ include "garage.metadataDir" . }}"
    data_dir = "{{ include "garage.dataDir" . }}"
    metadata_snapshots_dir = "{{ include "garage.snapshotsDir" . }}"

    # metadata_fsync IS OFF IN GARAGE'S DEFAULTS, and this is the single
    # setting most easily lost in a "tidy up the obvious defaults" pass. It is
    # set here on purpose: Longhorn's block layer cannot help if the engine
    # never calls fsync - a write that was never handed to the kernel is not in
    # any replica and not in any snapshot. Garage's own example configuration
    # pairs it with metadata_auto_snapshot_interval, as here.
    metadata_fsync = {{ $settings.metadataFsync }}
    # data_fsync stays off, which is Garage's default and the right trade for
    # the blocks: a lost block is one object, is detectable by a scrub, and is
    # replaceable from the sender's copy; a lost metadata write can take the
    # whole index with it. Turning it on costs an fsync per written block.
    data_fsync = {{ $settings.dataFsync }}

    # THE ONLY CONSISTENT RECOVERY POINT THIS DEPLOYMENT HAS. File-system
    # snapshots of metadata_dir guarantee nothing - Garage says so itself: a
    # snapshot taken during a metadata write is corrupt and useless as a
    # restore point, which is why Garage recommends ITS OWN snapshot mechanism
    # in all cases. This is the correction that reversed a premise of the
    # original plan: Longhorn backs up the blocks, not the metadata.
    #
    # KNOW THE WINDOW: Garage keeps exactly TWO snapshots (KEEP_SNAPSHOTS = 2),
    # the first one after interval/2 and every later one after interval times a
    # random 1.0-1.2. At "6h" that means the first after ~3h and a restore
    # window of roughly 6-14.4h afterwards. A corruption discovered later than
    # that is older than every snapshot that still exists.
    metadata_auto_snapshot_interval = "{{ $settings.metadataAutoSnapshotInterval }}"

    block_size = "{{ $settings.blockSize }}"
    # An integer is the zstd level; the literal "none" disables compression and
    # has to be QUOTED to stay valid TOML. Garage's deserializer takes exactly
    # those two forms - and 0 is not the second one: it is a valid integer that
    # zstd reads as its default level (3), so writing 0 to "switch compression
    # off" quietly turns it up.
    {{- if kindIs "string" $settings.compressionLevel }}
    compression_level = {{ $settings.compressionLevel | quote }}
    {{- else }}
    compression_level = {{ $settings.compressionLevel | int64 }}
    {{- end }}

    # Inter-node RPC. There are no other nodes here (no bootstrap_peers, no
    # kubernetes_discovery - see values.yaml), but Garage binds this listener
    # regardless and refuses to start without an rpc_secret, which arrives from
    # the environment.
    rpc_bind_addr = "{{ $settings.bindAddress }}:{{ $settings.ports.rpc | int64 }}"

    [s3_api]
    api_bind_addr = "{{ $settings.bindAddress }}:{{ $settings.ports.s3 | int64 }}"
    # The region name clients must use. It is not a cloud region, it is a label
    # both sides have to agree on - a consumer configured for a different
    # region gets a signature error, not a redirect.
    s3_region = "{{ $settings.s3.region }}"
    {{- with $settings.s3.rootDomain }}
    # Vhost-style bucket access (<bucket>.<root_domain>). Only useful with an
    # Ingress and a wildcard DNS entry in front of it; path-style access works
    # without it and is what an in-cluster consumer uses.
    root_domain = {{ . | quote }}
    {{- end }}

    {{- if $settings.web.enabled }}

    [s3_web]
    # Static website hosting, OFF by default (see values.yaml): it is a second
    # HTTP listener and serves nothing at all without a root_domain, while the
    # consumer this chart was built for speaks S3.
    bind_addr = "{{ $settings.bindAddress }}:{{ $settings.ports.web | int64 }}"
    {{- with $settings.web.rootDomain }}
    root_domain = {{ . | quote }}
    {{- end }}
    {{- end }}

    [admin]
    # Bound on 0.0.0.0 rather than [::] BECAUSE ALL THREE PROBES USE THIS PORT.
    # A dual-stack listener on [::] only accepts the kubelet's IPv4 connection
    # while net.ipv6.bindv6only is 0 - true by default on Linux, but a node
    # sysctl this chart cannot see, and the failure would be a pod that never
    # becomes Ready for a reason nothing in the manifest points at. The S3 and
    # RPC listeners keep the upstream [::] default; set garage.bindAddress to
    # "0.0.0.0" on an IPv4-only cluster, or garage.adminBindAddress to "[::]"
    # on an IPv6-only one.
    #
    # NO admin_token IS SET, so every authenticated admin endpoint is closed.
    # That is not an omission: /health, the one endpoint the probes use, is
    # explicitly token-free (@special in the admin router), and nothing in this
    # deployment calls the authenticated API - the CLI inside the pod talks RPC,
    # not HTTP. Adding a token would mean a fifth vault field guarding nothing
    # that is in use. If you need it, it is admin_token_file / the
    # GARAGE_ADMIN_TOKEN environment variable - and the file form carries the
    # same permission trap as rpc_secret_file (see values.yaml).
    api_bind_addr = "{{ $settings.adminBindAddress }}:{{ $settings.ports.admin | int64 }}"