{{- /* NetworkPolicy for garage (Gitea #5). Garage declares four ports and they split three ways, which makes this the second-clearest illustration of the convention after keycloak: s3 (3900) the S3 API. The Ingress points at it and no probe touches it, so it is RESTRICTED by source. web (3902) static website serving, same treatment as s3. admin (3903) the port all three probes address - tcpSocket twice and a httpGet /health - so it stays open inside the cluster. This is also the admin API, which is the uncomfortable part and is named in values.yaml: narrowing it means probeSources. rpc (3901) the node-to-node bus. It has NO Service at all, deliberately (see garage.service.yaml), and it gets the narrowest rule this object can express: the pods this policy itself selects. All four numbers are derived from garage.settings, so moving a port in values moves the policy with it. WHAT THIS OBJECT IS: a ceiling on reachability, not an authentication layer. Only the ports declared below are reachable at all - whatever else the process binds, now or after the next app update, is blocked without anyone having to notice it exists. WHAT IT IS NOT: - EGRESS IS COARSE. A NetworkPolicy matches CIDRs and selectors and NEVER hostnames, so "only to the database host" cannot be expressed at all. This cluster's CNI is Canal (Flannel plus Calico for policy) and has no FQDN rule; Calico's GlobalNetworkSet could express one but would bind this chart to Calico, the same argument that kept implementation-specific objects out of the Gateway-API sweep (#114). Each egress rule therefore permits its PORT to any destination. - NOT A BOUNDARY WHERE THE CNI DOES NOT ENFORCE ONE. An unenforced policy is accepted by the API server and filters nothing, so everything reads as configured while nothing is filtered. - THE REASONING IS IN TEMPLATE COMMENTS, not in "#" comments: those are part of the rendered object, travel into the cluster and appear in every release diff. ci/run-networkpolicy-test.sh fails on a rendered "#". THE PORT NUMBERS BELOW ARE POD PORTS, NEVER NODEPORTS. A NetworkPolicy is evaluated at the pod's network interface, after kube-proxy has already translated a NodePort to the targetPort, so a NodePort number here would match nothing while looking strict - and, being the only rule for that port, would block every inbound connection instead. ci/run-networkpolicy-test.sh asserts every port here against the containerPorts of the pod spec this policy selects. */ -}} {{- if .Values.networkPolicy.enabled }} {{- $settings := fromYaml (include "garage.settings" .) }} {{- $np := .Values.networkPolicy }} {{- $ns := .Release.Namespace }} {{- $probeSources := $np.probeSources | default list }} {{- $allowedNs := $np.allowedNamespaces | default list }} {{- $ic := $np.ingressController | default dict }} {{- $routed := or .Values.ingress.enabled .Values.httpRoute.enabled }} {{- /* networkPolicy.ingress - the policy's INBOUND side, and not the same thing as networkPolicy.ingressController (the SOURCE SELECTOR for the ingress controller's pods, a SIBLING of it and no part of it) nor as the top-level .Values.ingress (the Ingress object). All three match a search for "ingress"; one round of this work already mistook the second for an inbound escape hatch on the strength of the substring. */}} {{- $npIngress := $np.ingress | default dict }} {{- $ambientMesh := $npIngress.ambientMesh }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: {{ .Release.Name }}-{{ .Chart.Name }}-networkpolicy labels: app: {{ .Release.Name }}-{{ .Chart.Name }}-networkpolicy spec: {{- /* Selects the StatefulSet's pods. ci/run-networkpolicy-test.sh fails if this matches no pod spec of the same render - which is the mistake that would otherwise leave a policy in the namespace filtering nothing, and its usual cause is a label renamed in the workload template. */}} podSelector: matchLabels: app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs policyTypes: - Ingress {{- /* Egress is listed and it is narrow, which is the point: a storage node's only outbound traffic is DNS and its own cluster bus. Not permitted, and named in values.yaml because it is the plausible surprise: outbound HTTPS. Garage does not need it for S3 serving, but a deployment that configures remote replication or an external metrics sink does. */}} - Egress ingress: {{- /* THE S3 API AND THE STATIC WEB PORT, both genuinely restricted by source, because no probe addresses either of them: this release's own namespace, whatever allowedNamespaces adds, and the ingress controller. THE SOURCE LIST IS THE THING TO GET RIGHT HERE. Every consumer of this S3 endpoint that lives in another namespace - a backup job, a Zipline using it for uploads, a Stalwart blob store - is a pod in that namespace and is BLOCKED unless the namespace is in allowedNamespaces. Consumers reaching the published Ingress name instead arrive through the controller and are already covered. A blocked S3 consumer fails at the application level, with nothing naming this object. */}} - ports: - protocol: TCP port: {{ $settings.ports.s3 | int64 }} - protocol: TCP port: {{ $settings.ports.web | int64 }} from: - namespaceSelector: matchLabels: {{- /* Set by Kubernetes itself on every namespace (>= 1.21), so this needs no labelling work on a namespace this chart does not own. */}} kubernetes.io/metadata.name: {{ $ns }} {{- if and $routed $ic.namespace }} {{- /* ONE from-entry with TWO selectors, which is an AND: pods carrying these labels IN that namespace. Written as two list entries it would be an OR, and would open the port to every pod in kube-system. */}} - namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ $ic.namespace | quote }} {{- with $ic.podLabels }} podSelector: matchLabels: {{- toYaml . | nindent 14 }} {{- end }} {{- end }} {{- range $allowedNs }} - namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ . | quote }} {{- end }} {{- /* THE ADMIN PORT, and it is the probe path: liveness and startup open a TCP socket against it and readiness fetches /health from it. So it stays open inside the cluster by default. That is worth stating rather than glossing, because this is the ADMIN API: key management and cluster layout live here. The policy caps it at "reachable from inside the cluster", not at "closed", and probeSources is what closes the rest - at the usual price of needing the worker node IPs. */}} - ports: - protocol: TCP port: {{ $settings.ports.admin | int64 }} {{- if $ambientMesh }} {{- /* FIRST HALF OF networkPolicy.ingress.ambientMesh (Gitea #143). 15008/TCP is ztunnel's HBONE tunnel, the port every inbound connection arrives on once the namespace carries istio.io/dataplane-mode - ambient "does not and will never bypass Kubernetes NetworkPolicy enforcement", so without it a pod in the mesh is simply unreachable. It is the one port here that is NOT a containerPort of the selected pod spec: ztunnel terminates it, not the application. ci/run-networkpolicy- test.sh therefore holds a NAMED exception for exactly this number, gated on exactly this value - and assertion 7 of that script proves, on a second render, that 15008 stays the ONLY such port and that it is absent while the value is false. NOT MEASURED: that 15008 alone suffices comes from Istio's documentation. Measured on this cluster is only which listeners ztunnel opens. Provable at the first namespace to join the mesh; none has. */}} - protocol: TCP port: 15008 {{- end }} {{- if $probeSources }} {{- /* RESTRICTED, because the operator supplied probeSources. The kubelet's probes come from the IP of the WORKER NODE running the pod, so that address has to be in the list or the pod never becomes Ready - and a pod that is never Ready reads as a broken application rather than as a policy. No ingress-controller entry on this rule: nothing should route to the admin API from outside. */}} from: - namespaceSelector: matchLabels: {{- /* Set by Kubernetes itself on every namespace (>= 1.21), so this needs no labelling work on a namespace this chart does not own. */}} kubernetes.io/metadata.name: {{ $ns }} {{- range $allowedNs }} - namespaceSelector: matchLabels: kubernetes.io/metadata.name: {{ . | quote }} {{- end }} {{- range $probeSources }} - ipBlock: cidr: {{ . | quote }} {{- end }} {{- if $ambientMesh }} {{- /* SECOND HALF OF networkPolicy.ingress.ambientMesh, and the reason it is one value and not two. Inside the mesh the kubelet's probes are redirected and no longer arrive from the worker node's address: they arrive from 169.254.7.127. An operator who only opens 15008 by hand writes a policy that passes review - and the moment probeSources is non-empty every probe is dropped, the pod never becomes Ready, and the symptom reads as a broken application rather than as a policy. That is assertion 4 of ci/run-networkpolicy-test.sh one value further along. A switch whose halves can only be had together cannot be set halfway. Only in this branch, because this is the only branch with a "from" list: with no sources restricted the rule is open and needs no peer. The IPv6 counterpart is deliberately absent - this cluster is SingleStack IPv4, and an ipBlock of the wrong family is accepted and matches nothing. UNPROVEN IN OPERATION: probeSources is empty in every chart here, so no release reaches this branch. CI proves its render, through the overlays in ci/networkpolicy. */}} - ipBlock: cidr: 169.254.7.127/32 {{- end }} {{- else }} {{- /* NO "from" SELECTOR, WHICH MEANS OPEN - deliberately, and only inside the cluster, because this port carries no NodePort. The alternative was to guess the worker node CIDRs, and a wrong guess blocks every kubelet probe: the pod stays un-Ready, the rollout times out, and the cause looks like the application. An empty "from: []" would be worse than either - it allows everything too, but while reading as a restriction. Narrow it with networkPolicy.probeSources. */}} {{- end }} {{- /* THE NODE-TO-NODE RPC BUS, restricted to the pods this policy already selects - a podSelector with no namespaceSelector, which means "these labels, in this namespace". This port deliberately has no Service (garage.service.yaml says why: a Service port would advertise a cluster interface that is not one), so this rule is the only description of it anywhere in the chart. The matching egress rule below is what lets a node dial OUT to its peers. */}} - ports: - protocol: TCP port: {{ $settings.ports.rpc | int64 }} from: - podSelector: matchLabels: app: {{ .Release.Name }}-{{ .Chart.Name }}-sfs egress: {{- /* DNS, unconditionally. There is no value to switch this off, and that is deliberate: a knob whose only other setting breaks every name lookup in the pod is a trap and not a feature. */}} - ports: - protocol: UDP port: 53 - protocol: TCP port: 53 {{- /* Outbound RPC to the other nodes of this cluster. Needed as well as the inbound rule: policyTypes lists Egress, so a node dialling its peer is subject to this policy too, and with only the inbound rule the bus would be permitted in one direction - a layout that never converges while every node looks healthy. */}} - ports: - protocol: TCP port: {{ $settings.ports.rpc | int64 }} {{- with $np.egress }} {{- with .extraPorts }} {{- /* Ports only, to any destination - see the header. This is the operator's escape hatch from the egress cap, and the only alternative to switching the whole policy off. */}} - ports: {{- range . }} - protocol: TCP port: {{ int . }} {{- end }} {{- end }} {{- end }} {{- end }}