diff --git a/.github/workflows/install-test-all-charts.yml b/.github/workflows/install-test-all-charts.yml new file mode 100644 index 0000000..7840b44 --- /dev/null +++ b/.github/workflows/install-test-all-charts.yml @@ -0,0 +1,216 @@ +# The all-charts install-test: the same job the pull-request pipeline runs, with +# a matrix over every installable chart instead of the changed ones (Gitea #52). +# +# The question it answers is not "did this change break a chart" - that is the +# pull-request pipeline's job - but "does the environment under an UNCHANGED +# chart still hold". The install-test matrix follows the charts a pull request +# touches, so a chart nobody has edited since the forge move carries a green +# history from a different runner, and that green says nothing about today. +# +# It exists because the answer arrived once by accident: Gitea PR #48 corrected +# a comment in 21 charts, which made every one of them "changed", and so became +# the first all-charts run after the move - 19 of 20 matrix charts came up in +# 12m41s in one job, and exactly one did not (rss-bridge, Gitea #51). That +# result was worth having and nobody had planned it. This workflow makes it +# repeatable. +# +# WHEN TO RUN IT: after a change to the runner, to K3S_IMAGE, to the pinned +# KUBECTL_VERSION, to the fixtures under ci/ - and after a forge move. Not on a +# schedule: it would spend compute every day on a question that changes rarely, +# and a red run nobody asked for is a red run nobody reads. +name: Install-test (all charts) +on: + workflow_dispatch: + inputs: + charts: + required: false + type: string + default: "" + description: "charts to test, separated by spaces or commas - empty means every installable chart" +jobs: + select-charts: + name: "Select charts" + runs-on: ubuntu-latest + outputs: + charts: ${{ steps.select.outputs.charts }} + count: ${{ steps.select.outputs.count }} + steps: + - name: "Checkout Repository" + uses: actions/checkout@v7 + - name: "Select the charts to test" + id: select + env: + # The dispatch input is read as an environment variable and never + # interpolated into the script body. A "${{ inputs.charts }}" inside + # the shell would let whoever triggers the run write commands into + # this step; it is a small door and there is no reason to leave it + # open. The name check further down is the second half of that: these + # values end up as a matrix entry, a k3d cluster name and an argument + # to ci/run-install-test.sh. + REQUESTED: ${{ inputs.charts }} + run: | + FAIL=0 + + # --- The candidates ------------------------------------------------- + # Empty input: every directory with a Chart.yaml, which is the same + # definition detect-changed-charts uses for "this is a chart". No list + # to maintain - adding a chart enrols it, the way the quantity render + # test enrols one. + # + # A subset was asked for: exactly those names. This is what lets + # someone test a single chart against the runner, which is what Gitea + # #51 needs - and it is why an unknown or excluded name below is an + # error rather than a silent omission. Asking for one chart and + # getting a green run over zero charts would be the worst answer this + # workflow could give. + if [ -z "$(printf '%s' "$REQUESTED" | tr -d '[:space:],')" ]; then + SUBSET=false + CANDIDATES="$(ls -d -- */Chart.yaml 2>/dev/null | sed 's|/Chart.yaml$||' | sort)" + echo "No subset requested - every chart directory is a candidate." + else + SUBSET=true + CANDIDATES="$(printf '%s' "$REQUESTED" | tr ',' ' ' | tr -s '[:space:]' '\n' | grep -v '^$' | sort -u)" + echo "Requested subset: $(echo "$CANDIDATES" | tr '\n' ' ')" + fi + + for C in $CANDIDATES; do + case "$C" in + *[!a-z0-9-]*) + echo "::error::'$C' is not a chart name - chart directories in this repository are lowercase letters, digits and dashes." + FAIL=1 + continue + ;; + esac + if [ ! -f "$C/Chart.yaml" ]; then + echo "::error::'$C' is not a chart directory of this repository (no $C/Chart.yaml)." + FAIL=1 + fi + done + [ "$FAIL" -eq 0 ] || exit 1 + + CHARTS="$(printf '%s\n' $CANDIDATES | jq -R . | jq -sc 'unique')" + + # --- The exclusion list, through the one parser --------------------- + # ci/run-install-test.sh --list-excluded emits "" + # and is the only thing in this repository that reads the format of + # ci/excluded-charts.txt - here no more than in validate-charts.yml, + # because a second parser would be a second chance to select + # different charts, silently and in either direction (issue #231). + # The comparison is jq's "!=" on whole strings, so a chart name is + # never read as a pattern. + # + # The two callers differ in what they do with a hit, and on purpose. + # Over all charts, an excluded chart is dropped and named with its + # reason, exactly as in the pull-request pipeline. In a requested + # subset it is an ERROR: someone typed that name, and quietly + # returning a green run for a chart that was never attempted is the + # lie #231 removed from the check list. + SUMMARY_ROWS="" + while IFS=$'\t' read -r EX_NAME EX_REASON; do + [ -n "$EX_NAME" ] || continue + if [ "$(echo "$CHARTS" | jq -r --arg c "$EX_NAME" 'index($c) != null')" = "true" ]; then + if [ "$SUBSET" = true ]; then + echo "::error::chart '$EX_NAME' was requested but is listed in ci/excluded-charts.txt: $EX_REASON" + FAIL=1 + fi + # A "|" in the justification would break out of the table cell. + SUMMARY_ROWS="${SUMMARY_ROWS}| \`$EX_NAME\` | ${EX_REASON//|/\\|} |"$'\n' + echo "Excluded from the install-test: $EX_NAME - $EX_REASON" + fi + CHARTS="$(echo "$CHARTS" | jq -c --arg c "$EX_NAME" 'map(select(. != $c))')" + done <<< "$(ci/run-install-test.sh --list-excluded)" + [ "$FAIL" -eq 0 ] || exit 1 + + # An empty matrix does not fail - the install-test job would simply + # not be created and the run would end green, having tested nothing. + # That is the shape of signal this whole issue is about, so it is a + # loud error instead. + if [ "$CHARTS" = "[]" ]; then + echo "::error::no chart left to test - every candidate is listed in ci/excluded-charts.txt." + exit 1 + fi + + COUNT="$(echo "$CHARTS" | jq -r 'length')" + echo "charts=$CHARTS" >> "$GITHUB_OUTPUT" + echo "count=$COUNT" >> "$GITHUB_OUTPUT" + echo "Install-test matrix ($COUNT charts): $CHARTS" + + # --- The run record, first half ------------------------------------- + # What was asked for, when, and against what. The per-chart outcome + # and the environment each leg measured are written by the legs + # themselves (the called workflow's "Record the result" step), the + # verdict by the record job below. A green run nobody notes is + # unknown again next time. + { + echo "## All-charts install-test" + echo + echo "- Started: **$(date -u '+%Y-%m-%d %H:%M') UTC**" + echo "- Requested: $( [ "$SUBSET" = true ] && echo "subset \`$REQUESTED\`" || echo "**every installable chart**" )" + echo "- Matrix: **$COUNT charts** — $(echo "$CHARTS" | jq -r 'map("`" + . + "`") | join(", ")')" + echo "- Expected runtime, so a deviation is visible as one: **20 charts in 12m41s**, measured on Gitea PR #48 — one job, all legs concurrently on one Docker daemon. The job timeout is 30 minutes per leg." + echo "- One throwaway k3d cluster per chart, no reuse: a reused cluster would carry leftovers, a warm image cache and fixtures already applied, and \"came up from nothing\" is the statement this test exists to make." + echo + echo "Each chart's outcome, with the runner label, \`K3S_IMAGE\` and \`kubectl\` version it was measured against, is in the install-test job summaries; the verdict over the whole matrix is at the end." + echo + } >> "$GITHUB_STEP_SUMMARY" + + if [ -n "$SUMMARY_ROWS" ]; then + { + echo "### Charts excluded from the install-test" + echo + echo "These charts are listed in \`ci/excluded-charts.txt\` and get **no** install-test job. There is no green tick for them, because nothing was attempted (issue #231)." + echo + echo "| Chart | Documented reason |" + echo "| --- | --- |" + printf '%s' "$SUMMARY_ROWS" + echo + } >> "$GITHUB_STEP_SUMMARY" + fi + + # The same workflow validate-charts.yml calls, with the same inputs, differing + # only in the chart list. Not a copy of the job: see the header of + # .github/workflows/install-test.yml for what would drift. + all-charts: + name: "All charts" + needs: select-charts + uses: ./.github/workflows/install-test.yml + with: + charts: ${{ needs.select-charts.outputs.charts }} + secrets: inherit + + # The verdict over the matrix as a whole. A matrix leg only ever sees its own + # job summary, so this is the one place that can say "all of them" - and the + # aggregate is what the question "has the environment changed" is answered + # with. It runs on failure too, hence "if: always()". + record: + name: "Record the run" + needs: [select-charts, all-charts] + if: always() + runs-on: ubuntu-latest + steps: + - name: "Write the verdict" + env: + RESULT: ${{ needs.all-charts.result }} + COUNT: ${{ needs.select-charts.outputs.count }} + run: | + case "$RESULT" in + success) + VERDICT="**all $COUNT charts installed and became Ready** on this runner, on $(date -u '+%Y-%m-%d')" + ;; + failure) + VERDICT="**at least one chart did not come up.** Which ones, and whether the chart failed or the cluster never came up, is in the per-chart records above. That is a result and not an accident to re-run: rss-bridge failing this way on Gitea PR #48 is why Gitea #51 exists" + ;; + skipped|cancelled) + VERDICT="**no verdict** - the matrix did not run ($RESULT). Nothing was tested; do not read this run as a green one" + ;; + *) + VERDICT="**no verdict** - unexpected matrix result '$RESULT'" + ;; + esac + { + echo "## Verdict" + echo + echo "$VERDICT" + echo + echo "Worth writing down where it belongs: a result from this run is what the next \"has anything drifted\" question is compared against. If it moved a chart forward or backward, say so on the chart's wiki page or in \`Lessons-Learned\`." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/install-test.yml b/.github/workflows/install-test.yml new file mode 100644 index 0000000..3f5750c --- /dev/null +++ b/.github/workflows/install-test.yml @@ -0,0 +1,414 @@ +# The install-test, as one job with two callers. +# +# Both callers install a chart the SAME way; they differ only in which charts +# they name. validate-charts.yml passes the charts a pull request changed; +# install-test-all-charts.yml passes every installable chart on demand, to ask +# whether the environment underneath an UNCHANGED chart still holds (Gitea +# #52) - a chart untouched since the forge move carries a green history from a +# different runner, and that says nothing about today. +# +# It is a called workflow and not a second job, for the same reason +# ci/excluded-charts.txt has exactly one parser: a second copy is a second +# chance to test DIFFERENTLY, unnoticed. The parts that would drift are worth +# naming, because each was paid for once already - the pinned kubectl with its +# skew check, envsubst, the detection of a containerised job, the per-leg +# cluster name. A copy that slowly disagrees about any of them does not go red; +# it turns a green run into a statement about something other than what was +# assumed. +# +# What stays with the callers: WHICH charts are tested, and whether the job +# runs at all (needs/if). Everything that decides HOW one is installed is here. +name: Install-test +on: + workflow_call: + inputs: + charts: + required: true + type: string + description: 'JSON array of chart names, e.g. ["cyberchef","zipline"]. The entries of ci/excluded-charts.txt must already be gone - see the matrix comment below.' + secrets: + OP_CONNECT_TOKEN: + # Needed by exactly one leg - freesailarr, for the 1Password Connect + # lookup of its CI Wireguard key. Declared OPTIONAL on purpose, where + # publish-helm-chart.yml declares its token required: on this instance a + # missing secret does not fail the job, it fails job CREATION, so the + # run shows no steps at all (README, "Where this repo lives"). With + # required: true a pull request from a fork - which carries no secrets - + # would lose the entire install-test that way, every chart at once, with + # nothing naming the cause. Optional instead: only the freesailarr leg + # fails, loudly, at the ":?" guards that name the missing op:// field. + required: false + description: "1Password Connect token; only the freesailarr leg reads it" +jobs: + install-test: + name: "Install-test ${{ matrix.chart }}" + # The one job in this repo that does not simply say ubuntu-latest, and the + # expression is not decoration - the same label means different machines on + # the two forges this repo currently lives on (#225). + # + # This is the only job that needs a Docker socket: it creates a k3d + # cluster, and k3d is a Docker client. On GitHub's hosted runners every + # label carries Docker, so the question never came up. On the Gitea + # instance it was MEASURED on a probe pull request, per job: + # detect-changed-charts, build-chart, the quantity render and the Trivy + # scan all pass on `ubuntu-latest` - none of them needs Docker - while the + # install-test dies there at "Create k3d cluster" with "Cannot connect to + # the Docker daemon at unix:///var/run/docker.sock", an error that points + # at k3d and never at the runner label. `ubuntu-docker` is the label that + # carries a socket, and Gitea does evaluate this expression: the probe job + # was picked up by the docker runner with labels ["ubuntu-docker"]. + # + # Why an expression and not a plain `ubuntu-docker`: that label does not + # exist on GitHub, so every chart PR would sit queued forever for as long + # as this repo still builds on github.com - blocking the merge with no + # error anywhere. github.server_url is the one context value that separates + # the two without naming our instance. + # + # The label is necessary and was, for a while, not sufficient: + # `ubuntu-docker`'s daemon reported "Cgroup Driver: none" plus "Running in + # rootless-mode without cgroups", and k3d got as far as starting the k3s + # node, which restarted immediately with an empty log because k3s needs + # cgroup delegation (#225). Nothing in this file addressed that - it was + # dealt with on the instance side. It no longer blocks, and the evidence is + # Gitea PR #48: 19 of 20 matrix charts installed and became Ready on this + # label. The twentieth, rss-bridge, failed and is Gitea #51. + runs-on: ${{ github.server_url == 'https://github.com' && 'ubuntu-latest' || 'ubuntu-docker' }} + # Blocking since Renovate automerges patch/minor updates: a chart that no + # longer starts must stop the PR before it reaches main and publishes. This + # ends the stabilization phase of issue #22 - the job reported failures + # without blocking while it proved itself across several chart PRs. + # + # The 30 minutes are per leg and hold for the all-charts caller too, with + # the measured figure to compare against: Gitea PR #48 ran 20 charts in + # 12m41s in one job, all legs concurrently on one Docker daemon. If a run + # ever comes near this timeout, it is raised HERE, once, for both callers - + # a value that differs per caller would mean the two tests give up at + # different points while claiming to be the same test. + timeout-minutes: 30 + strategy: + fail-fast: false + # The charts arrive from the caller ALREADY filtered against + # ci/excluded-charts.txt: validate-charts.yml does it in + # detect-changed-charts, install-test-all-charts.yml in select-charts, and + # both go through ci/run-install-test.sh --list-excluded, the one parser + # of that file. Both also name every chart they dropped WITH its + # documented reason in their own job summary, because an absent matrix + # entry says "not run" and never why (issue #231). + # + # DO NOT move that decision in here, e.g. as an "if" on this job + # if: ... && !contains(fromJSON(inputs.excluded-charts), matrix.chart) + # to get a visible "Skipped" per chart instead of an absent entry. + # GitHub rejects it - jobs..if is evaluated BEFORE the matrix is + # expanded, so the matrix context does not exist there: + # context "matrix" is not allowed here. + # available contexts are "github", "inputs", "needs", "vars". + # Measured on run 33650724109 (PR #233, closed): the whole workflow ended + # in failure with zero jobs created. The per-chart value has to be gone + # from the matrix before this workflow is called. + matrix: + chart: ${{ fromJSON(inputs.charts) }} + env: + K3D_VERSION: v5.9.0 + K3S_IMAGE: docker.io/rancher/k3s:v1.35.5-k3s1 + # Pinned rather than taken from the runner image. The argument is not + # "Gitea's image has no kubectl" - that is only how it was noticed. A + # kubectl inherited from whatever the image happens to ship drifts + # unannounced with every image rebuild, and an install-test is supposed + # to run against a KNOWN client. That GitHub's hosted image already + # carries one does not make the pin unnecessary; it only makes the drift + # invisible there. The value mirrors K3S_IMAGE above (v1.35.5), which is + # the cluster this client talks to. + # renovate: datasource=github-releases depName=kubernetes/kubernetes + KUBECTL_VERSION: v1.35.5 + # The same expression as runs-on above, written a second time because the + # env context is not available in runs-on - so it cannot be the other way + # round, and reading the label back off the runner is not possible either + # (RUNNER_NAME below names the machine, not the label that picked it). + # It exists for the run record at the end of this job: "on which runner + # was this green tick measured" is the whole question of Gitea #52. Kept + # right under runs-on, so a change to one that forgets the other is + # visible on a single screen. + RUNNER_LABEL: ${{ github.server_url == 'https://github.com' && 'ubuntu-latest' || 'ubuntu-docker' }} + steps: + - name: "Checkout Repository" + uses: actions/checkout@v7 + - name: "Install Helm" + uses: azure/setup-helm@v5 + - name: "Install kubectl" + run: | + # kubectl and the cluster are a pair: the client talks to the server + # K3S_IMAGE starts, and Kubernetes supports at most one minor of + # skew between them. Renovate updates KUBECTL_VERSION on its own + # while K3S_IMAGE is tracked by nothing, so without this check the + # two drift apart until the client is talking to a server it no + # longer supports - silently, because kubectl only warns. And it is + # not theoretical: measured by dry run, a lookup from v1.34.0 + # offers v1.37.0 in ONE step, because Renovate fills the non-major + # bucket with the newest candidate rather than walking the minors. + # So this check WILL fire one day, on purpose, with a message that + # says which of the two pins to move. + # + # Deliberately not solved with an allowedVersions constraint pinning + # kubectl to the k3s minor: that is the nginx trap the README + # describes - a filter that silently stops offering updates the day + # the other side moves. A loud failure beats a quiet freeze. + read -r KMAJ KMIN <<<"$(printf '%s' "$KUBECTL_VERSION" | sed -E 's/^v([0-9]+)\.([0-9]+).*/\1 \2/')" + read -r SMAJ SMIN <<<"$(printf '%s' "$K3S_IMAGE" | sed -E 's/.*:v([0-9]+)\.([0-9]+).*/\1 \2/')" + SKEW=$(( (KMAJ * 100 + KMIN) - (SMAJ * 100 + SMIN) )) + if [ "${SKEW#-}" -gt 1 ]; then + echo "::error::kubectl $KUBECTL_VERSION and k3s $K3S_IMAGE are more than one minor apart. Bump K3S_IMAGE to match, or pin KUBECTL_VERSION back." + exit 1 + fi + curl -sSfL -o /tmp/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + mkdir -p "$HOME/.local/bin" + install -m 0755 /tmp/kubectl "$HOME/.local/bin/kubectl" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + "$HOME/.local/bin/kubectl" version --client + - name: "Provide envsubst" + run: | + # ci/run-install-test.sh renders ci//fixtures.env.yaml with + # envsubst - today only freesailarr has one, and without the tool + # the fixtures collapse to "error: no objects passed to apply", + # gluetun never gets its Wireguard key, and nothing behind its kill + # switch becomes Ready. GitHub's hosted image ships gettext-base, + # the Gitea runner image does not. The script is fine as it is; a + # standard tool is simply missing. + # + # CHECKED FOR, NOT PINNED - and that is a different decision from + # kubectl above, on purpose. kubectl's VERSION changes what the test + # means: its distance to the API server is a real source of failure, + # which is why it is pinned, tracked and guarded. envsubst + # substitutes text; which release performs the substitution does not + # change the result. Pinning it would buy nothing and add a line for + # Renovate to carry. So: present - do nothing, which keeps the + # GitHub path free of any package install; missing - fetch it, and + # the difference between the two runners is stated rather than + # hidden behind an unconditional apt-get. + if command -v envsubst >/dev/null 2>&1; then + echo "envsubst present at $(command -v envsubst) - nothing to do" + exit 0 + fi + echo "envsubst missing on this runner image - installing gettext-base" + if [ "$(id -u)" -eq 0 ]; then SUDO=""; else SUDO="sudo"; fi + $SUDO apt-get update -qq + $SUDO apt-get install -y -qq --no-install-recommends gettext-base + command -v envsubst + # freesailarr is the one chart whose install-test needs credentials: + # without a real Wireguard key gluetun never completes a handshake, and + # behind its kill switch not a single app container becomes Ready. The key + # comes from 1Password Connect like the registry credentials in + # publish-helm-chart.yml, and it is a CI-ONLY Mullvad key: Mullvad ties an + # address to a key and sends the tunnel to the endpoint it saw last, so + # reusing the production key would let every PR run pull the tunnel out + # from under the running stack - with the symptom in the cluster and the + # cause in a pull request. Loaded only for this chart; no other chart run + # touches the VPN key. + - name: "Configure OP-Connect" + if: matrix.chart == 'freesailarr' + uses: 1password/load-secrets-action/configure@v5 + with: + connect-host: https://op-connect.jk-dev.de + - name: "Load freesailarr VPN credentials" + id: load-freesailarr-vpn + if: matrix.chart == 'freesailarr' + uses: 1password/load-secrets-action@v5 + env: + OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} + FREESAILARR_WG_KEY: op://GitHub-Actions/freesailarr-mullvad-test-config/credential + FREESAILARR_WG_ADDRESSES: op://GitHub-Actions/freesailarr-mullvad-test-config/ip-address + # The same trap publish-helm-chart.yml documents at length: v5 of + # load-secrets-action does NOT export the loaded values as environment + # variables for later steps, only its step outputs carry them, and the + # loading step logs "Populating variable" and stays green either way. So + # the values are mapped into each step's own env block via + # steps..outputs.* and read as quoted shell variables. The ":?" guards + # run before the cluster is even created and name the op:// field that is + # missing - an empty key would otherwise reach gluetun and look like a + # broken chart instead of a missing secret. They print the variable name, + # never its value. + - name: "Verify the VPN credentials arrived" + if: matrix.chart == 'freesailarr' + env: + FREESAILARR_WG_KEY: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_KEY }} + FREESAILARR_WG_ADDRESSES: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_ADDRESSES }} + run: |- + : "${FREESAILARR_WG_KEY:?empty - check op://GitHub-Actions/freesailarr-mullvad-test-config/credential}" + : "${FREESAILARR_WG_ADDRESSES:?empty - check op://GitHub-Actions/freesailarr-mullvad-test-config/ip-address}" + echo "Both VPN values arrived non-empty." + - name: "Install k3d" + run: curl -sSfL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG="$K3D_VERSION" bash + # One cluster per matrix leg, named after the chart. It used to be the + # fixed name "install-test", and that was wrong on GitHub too - it just + # could not show there, because GitHub gives every matrix leg its own VM + # with its own Docker daemon, so two legs never saw each other's + # cluster. The environment hid the bug for two years rather than + # preventing it. Gitea runs all legs of a matrix CONCURRENTLY in ONE job + # on ONE daemon, and there the fixed name collides at once: measured on + # the probe PR of #225, one leg logs "Re-using existing network + # 'k3d-install-test'", the other dies on "The container name + # /k3d-install-test-server-0 is already in use" - and then the first + # leg's rollback deletes the shared network out from under the second, + # which fails with "network k3d-install-test not found". Both legs red, + # neither cause in either message. DO NOT put the fixed name back. + # + # Why the bare chart name and not a prefixed one: k3d rejects a cluster + # name over 32 characters ("Cluster name must be <= 32 characters", + # measured against k3d v5.9.0 - 32 passes, 33 fails), and every + # container is named k3d--server-0 from it. The longest chart + # here is calibre-web-automated at 21, so "install-test-" would + # be 34 and break exactly that one chart and no other - the kind of + # prefix that is worse than the fixed name because it only sometimes + # breaks. The guard below states the limit instead of leaving it to be + # rediscovered. + - name: "Create k3d cluster" + run: | + CLUSTER="${{ matrix.chart }}" + if [ "${#CLUSTER}" -gt 32 ]; then + echo "::error::chart name '$CLUSTER' is ${#CLUSTER} characters and k3d allows at most 32 for a cluster name." + exit 1 + fi + + # Does THIS job run inside a container? That, and not which forge we + # are on, is what decides whether k3d's cluster is reachable (#227). + # + # k3d publishes the API server port on the Docker *host*. When the + # job runs straight on the host - GitHub's hosted runners - that is + # the same machine, and the "https://0.0.0.0:" k3d writes into + # the kubeconfig works. When the job itself is a container, as every + # Gitea Actions job is, the host is the dind container and 0.0.0.0 + # points at the job container itself, where nothing listens. The + # cluster comes up fine and the first kubectl call is refused. + # + # Deliberately NOT keyed on github.server_url, although that would + # pick the right branch today: a self-hosted GitHub runner in a + # container has exactly this problem, and a condition that says + # "GitHub" while meaning "not containerised" is the kind of lie this + # repo exists to remove. So the state is detected instead of + # inferred, and the detection is the same question the fix needs + # answered anyway: ask the daemon about ourselves. Inside a + # container that the daemon knows, this returns our network and k3d + # joins it, which puts k3s and this job on one bridge where Docker's + # own DNS resolves the container names. On a host it returns nothing, + # because the hostname is not a container, and the old path is taken + # unchanged. + # The "|| true" is load-bearing: this step runs under `bash -e -o + # pipefail`, and on a host `docker inspect` exits non-zero, which + # would abort the step instead of taking the else branch. + JOB_NET="$(docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$(hostname)" 2>/dev/null | awk '{print $1}' || true)" + + if [ -n "$JOB_NET" ]; then + echo "Containerised job: $(hostname) on docker network '$JOB_NET' - k3d joins it." + k3d cluster create "$CLUSTER" --image "$K3S_IMAGE" --network "$JOB_NET" --wait --timeout 180s + # Same reason: the published port is useless from in here, so the + # kubeconfig is pointed at the load balancer container by name, + # over the shared network. If the API server certificate did not + # carry that name this would fail on the TLS handshake and the + # cluster-info below would say so - it holds without an extra + # --tls-san, measured on the probe PR of #225. + kubectl config set-cluster "k3d-$CLUSTER" --server="https://k3d-$CLUSTER-serverlb:6443" + else + echo "Job runs on the docker host - k3d's published port is reachable as written." + k3d cluster create "$CLUSTER" --image "$K3S_IMAGE" --wait --timeout 180s + fi + + # Reachability is proven HERE, and the shape of this check is the + # product of getting it wrong twice. Both mistakes are written down + # because both look like something to simplify away. + # + # It is a LOOP and not a single call. k3d's --wait returns when the + # node container reports ready, but the API server can still answer + # discovery with 503 for a moment after that. Measured: a single + # `kubectl cluster-info` right here failed with "ServiceUnavailable: + # the server is currently unable to handle the request" on one of + # two legs of the SAME GitHub run while the other passed. A check + # that is itself racy produces exactly the sporadic red this job + # must never have (#227) - and a test people have learned to re-run + # is one they have stopped believing. + # + # It counts READY NODES and not "did kubectl answer". `kubectl get + # nodes` exits 0 on an EMPTY list, so the first version passed + # against a cluster whose API server was up and whose node had never + # registered: the log said "No resources found", the step went + # green, and the failure surfaced five minutes later as a pod stuck + # in Pending with no node and no scheduling event - nowhere near its + # cause. A check that a broken cluster passes is worse than no + # check. `kubectl wait --for=condition=Ready nodes --all` is not the + # shorter way to write this: it has the identical hole, because with + # zero nodes the condition is vacuously satisfied. + # + # The evidence that this version earns its keep: only after it + # counted nodes did the real cause become visible in the dump below + # - k3s could not start containerd ("error creating fsnotify + # watcher: too many open files", an inotify limit on the host), and + # nothing above this line had ever said so. + for ATTEMPT in $(seq 1 90); do + READY_NODES="$(kubectl get nodes --no-headers 2>/dev/null | awk '$2 == "Ready"' | wc -l | tr -d ' ')" + [ "$READY_NODES" -ge 1 ] && break + if [ "$ATTEMPT" -eq 90 ]; then + echo "::error::the k3d cluster '$CLUSTER' answers but has no Ready node after 180s" + kubectl get nodes -o wide || true + kubectl describe nodes || true + echo "--- k3s server log ---" + docker logs "k3d-$CLUSTER-server-0" 2>&1 | tail -80 || true + exit 1 + fi + sleep 2 + done + kubectl get nodes -o wide + # Every chart reaching this step is one the exclusion filter let through, + # so the script's SKIP branch is never taken here - it exists for local + # runs. The variables below are empty for every chart but the one that + # declares them in ci//settings.env (REQUIRED_ENV); the script + # fails loudly if a declared one is missing, it never skips over it. + - name: "Install chart and wait for readiness" + # The id is read by the run record at the end of this job: its outcome + # is what separates "the chart did not come up" from "the cluster never + # came up", and the two mean opposite things for Gitea #52. + id: install + env: + FREESAILARR_WG_KEY: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_KEY }} + FREESAILARR_WG_ADDRESSES: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_ADDRESSES }} + run: ci/run-install-test.sh "${{ matrix.chart }}" + # Must carry the same per-leg name as the create step above: a blanket + # cleanup, or the old fixed name, would tear down the neighbouring leg's + # cluster on a runner that shares one Docker daemon - which is precisely + # the half of the collision that turned one red leg into two. + - name: "Delete k3d cluster" + if: always() + run: k3d cluster delete "${{ matrix.chart }}" || true + # The run record (Gitea #52). A green install-test says "this chart came + # up" and nothing about WHERE, and where is exactly what the question + # needs: a chart that has not been touched since the forge move carries a + # green history from another environment. So every leg writes down the + # four facts that decide what its result means - date, runner label, + # K3S_IMAGE, kubectl - next to its own outcome. A green run nobody notes + # is unknown again next time. + # + # Per leg, and not one table for the whole matrix, because there is no + # place to write one: a matrix leg only ever sees its own + # GITHUB_STEP_SUMMARY, on either forge. The aggregate verdict is written + # by the calling workflow, which does see the matrix as a whole. + # + # It reports steps.install.outcome and not the job status: a leg that died + # in "Create k3d cluster" never installed anything, and recording that as + # a failed chart would blame the chart for the environment - precisely the + # confusion this issue is about. + - name: "Record the result" + if: always() + env: + CHART: ${{ matrix.chart }} + INSTALL_OUTCOME: ${{ steps.install.outcome }} + run: | + case "$INSTALL_OUTCOME" in + success) RESULT="installed, every workload Ready" ;; + failure) RESULT="**FAILED** - the chart did not come up" ;; + *) RESULT="**FAILED before the install** - cluster or tooling, not the chart (install step outcome: '${INSTALL_OUTCOME:-none}')" ;; + esac + { + echo "### Install-test \`$CHART\`: $RESULT" + echo + echo "| Date (UTC) | Runner label | Runner name | K3S_IMAGE | kubectl |" + echo "| --- | --- | --- | --- | --- |" + echo "| $(date -u '+%Y-%m-%d %H:%M') | \`$RUNNER_LABEL\` | \`${RUNNER_NAME:-unknown}\` | \`$K3S_IMAGE\` | \`$KUBECTL_VERSION\` |" + echo + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/validate-charts.yml b/.github/workflows/validate-charts.yml index 11a85ef..75589a7 100644 --- a/.github/workflows/validate-charts.yml +++ b/.github/workflows/validate-charts.yml @@ -228,314 +228,30 @@ jobs: run: ci/run-trivy-config.sh "${{ matrix.chart }}" install-test: - name: "Install-test ${{ matrix.chart }}" + name: "Install-test" needs: [detect-changed-charts, build-chart] if: needs.detect-changed-charts.outputs.any-install == 'true' - # The one job in this repo that does not simply say ubuntu-latest, and the - # expression is not decoration - the same label means different machines on - # the two forges this repo currently lives on (#225). + # The job itself lives in .github/workflows/install-test.yml and is called + # from here and from install-test-all-charts.yml (Gitea #52). Everything + # that decides HOW a chart is installed moved there unchanged - the runner + # label expression, the pinned kubectl with its skew check, envsubst, the + # containerised-job detection, the per-leg cluster name, the freesailarr VPN + # credentials. What stayed here is WHICH charts are tested and whether the + # job runs at all. A copied job would be a second chance to test + # differently, unnoticed, which is the same argument that gives + # ci/excluded-charts.txt exactly one parser. # - # This is the only job that needs a Docker socket: it creates a k3d - # cluster, and k3d is a Docker client. On GitHub's hosted runners every - # label carries Docker, so the question never came up. On the Gitea - # instance it was MEASURED on a probe pull request, per job: - # detect-changed-charts, build-chart, the quantity render and the Trivy - # scan all pass on `ubuntu-latest` - none of them needs Docker - while the - # install-test dies there at "Create k3d cluster" with "Cannot connect to - # the Docker daemon at unix:///var/run/docker.sock", an error that points - # at k3d and never at the runner label. `ubuntu-docker` is the label that - # carries a socket, and Gitea does evaluate this expression: the probe job - # was picked up by the docker runner with labels ["ubuntu-docker"]. - # - # Why an expression and not a plain `ubuntu-docker`: that label does not - # exist on GitHub, so every chart PR would sit queued forever for as long - # as this repo still builds on github.com - blocking the merge with no - # error anywhere. github.server_url is the one context value that separates - # the two without naming our instance. - # - # KNOWN, and not fixed by this line: `ubuntu-docker`'s daemon is ROOTLESS - # and reports "Cgroup Driver: none" plus "Running in rootless-mode without - # cgroups". k3d then gets as far as starting the k3s node, which restarts - # immediately with an empty log, because k3s needs cgroup delegation. Until - # the instance's Docker-in-Docker provides it, this job cannot pass on - # Gitea - see #225. The label is necessary, it is just not sufficient. - runs-on: ${{ github.server_url == 'https://github.com' && 'ubuntu-latest' || 'ubuntu-docker' }} - # Blocking since Renovate automerges patch/minor updates: a chart that no - # longer starts must stop the PR before it reaches main and publishes. This - # ends the stabilization phase of issue #22 - the job reported failures - # without blocking while it proved itself across several chart PRs. - timeout-minutes: 30 - strategy: - fail-fast: false - # "install-charts", not "charts": the changed charts minus the ones in - # ci/excluded-charts.txt, filtered in detect-changed-charts. Excluded - # charts are named with their reason in that job's summary (issue #231). - # - # DO NOT try to move that decision into an "if" on this job, e.g. - # if: ... && !contains(fromJSON(needs.detect-changed-charts.outputs.excluded-charts), matrix.chart) - # to get a visible "Skipped" per chart instead of an absent entry. - # GitHub rejects it - jobs..if is evaluated BEFORE the matrix is - # expanded, so the matrix context does not exist there: - # context "matrix" is not allowed here. - # available contexts are "github", "inputs", "needs", "vars". - # Measured on run 33650724109 (PR #233, closed): the whole workflow ended - # in failure with zero jobs created. The per-chart value has to be gone - # from the matrix by the time this job is reached. - matrix: - chart: ${{ fromJSON(needs.detect-changed-charts.outputs.install-charts) }} - env: - K3D_VERSION: v5.9.0 - K3S_IMAGE: docker.io/rancher/k3s:v1.35.5-k3s1 - # Pinned rather than taken from the runner image. The argument is not - # "Gitea's image has no kubectl" - that is only how it was noticed. A - # kubectl inherited from whatever the image happens to ship drifts - # unannounced with every image rebuild, and an install-test is supposed - # to run against a KNOWN client. That GitHub's hosted image already - # carries one does not make the pin unnecessary; it only makes the drift - # invisible there. The value mirrors K3S_IMAGE above (v1.35.5), which is - # the cluster this client talks to. - # renovate: datasource=github-releases depName=kubernetes/kubernetes - KUBECTL_VERSION: v1.35.5 - steps: - - name: "Checkout Repository" - uses: actions/checkout@v7 - - name: "Install Helm" - uses: azure/setup-helm@v5 - - name: "Install kubectl" - run: | - # kubectl and the cluster are a pair: the client talks to the server - # K3S_IMAGE starts, and Kubernetes supports at most one minor of - # skew between them. Renovate updates KUBECTL_VERSION on its own - # while K3S_IMAGE is tracked by nothing, so without this check the - # two drift apart until the client is talking to a server it no - # longer supports - silently, because kubectl only warns. And it is - # not theoretical: measured by dry run, a lookup from v1.34.0 - # offers v1.37.0 in ONE step, because Renovate fills the non-major - # bucket with the newest candidate rather than walking the minors. - # So this check WILL fire one day, on purpose, with a message that - # says which of the two pins to move. - # - # Deliberately not solved with an allowedVersions constraint pinning - # kubectl to the k3s minor: that is the nginx trap the README - # describes - a filter that silently stops offering updates the day - # the other side moves. A loud failure beats a quiet freeze. - read -r KMAJ KMIN <<<"$(printf '%s' "$KUBECTL_VERSION" | sed -E 's/^v([0-9]+)\.([0-9]+).*/\1 \2/')" - read -r SMAJ SMIN <<<"$(printf '%s' "$K3S_IMAGE" | sed -E 's/.*:v([0-9]+)\.([0-9]+).*/\1 \2/')" - SKEW=$(( (KMAJ * 100 + KMIN) - (SMAJ * 100 + SMIN) )) - if [ "${SKEW#-}" -gt 1 ]; then - echo "::error::kubectl $KUBECTL_VERSION and k3s $K3S_IMAGE are more than one minor apart. Bump K3S_IMAGE to match, or pin KUBECTL_VERSION back." - exit 1 - fi - curl -sSfL -o /tmp/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - mkdir -p "$HOME/.local/bin" - install -m 0755 /tmp/kubectl "$HOME/.local/bin/kubectl" - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - "$HOME/.local/bin/kubectl" version --client - - name: "Provide envsubst" - run: | - # ci/run-install-test.sh renders ci//fixtures.env.yaml with - # envsubst - today only freesailarr has one, and without the tool - # the fixtures collapse to "error: no objects passed to apply", - # gluetun never gets its Wireguard key, and nothing behind its kill - # switch becomes Ready. GitHub's hosted image ships gettext-base, - # the Gitea runner image does not. The script is fine as it is; a - # standard tool is simply missing. - # - # CHECKED FOR, NOT PINNED - and that is a different decision from - # kubectl above, on purpose. kubectl's VERSION changes what the test - # means: its distance to the API server is a real source of failure, - # which is why it is pinned, tracked and guarded. envsubst - # substitutes text; which release performs the substitution does not - # change the result. Pinning it would buy nothing and add a line for - # Renovate to carry. So: present - do nothing, which keeps the - # GitHub path free of any package install; missing - fetch it, and - # the difference between the two runners is stated rather than - # hidden behind an unconditional apt-get. - if command -v envsubst >/dev/null 2>&1; then - echo "envsubst present at $(command -v envsubst) - nothing to do" - exit 0 - fi - echo "envsubst missing on this runner image - installing gettext-base" - if [ "$(id -u)" -eq 0 ]; then SUDO=""; else SUDO="sudo"; fi - $SUDO apt-get update -qq - $SUDO apt-get install -y -qq --no-install-recommends gettext-base - command -v envsubst - # freesailarr is the one chart whose install-test needs credentials: - # without a real Wireguard key gluetun never completes a handshake, and - # behind its kill switch not a single app container becomes Ready. The key - # comes from 1Password Connect like the registry credentials in - # publish-helm-chart.yml, and it is a CI-ONLY Mullvad key: Mullvad ties an - # address to a key and sends the tunnel to the endpoint it saw last, so - # reusing the production key would let every PR run pull the tunnel out - # from under the running stack - with the symptom in the cluster and the - # cause in a pull request. Loaded only for this chart; no other chart run - # touches the VPN key. - - name: "Configure OP-Connect" - if: matrix.chart == 'freesailarr' - uses: 1password/load-secrets-action/configure@v5 - with: - connect-host: https://op-connect.jk-dev.de - - name: "Load freesailarr VPN credentials" - id: load-freesailarr-vpn - if: matrix.chart == 'freesailarr' - uses: 1password/load-secrets-action@v5 - env: - OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} - FREESAILARR_WG_KEY: op://GitHub-Actions/freesailarr-mullvad-test-config/credential - FREESAILARR_WG_ADDRESSES: op://GitHub-Actions/freesailarr-mullvad-test-config/ip-address - # The same trap publish-helm-chart.yml documents at length: v5 of - # load-secrets-action does NOT export the loaded values as environment - # variables for later steps, only its step outputs carry them, and the - # loading step logs "Populating variable" and stays green either way. So - # the values are mapped into each step's own env block via - # steps..outputs.* and read as quoted shell variables. The ":?" guards - # run before the cluster is even created and name the op:// field that is - # missing - an empty key would otherwise reach gluetun and look like a - # broken chart instead of a missing secret. They print the variable name, - # never its value. - - name: "Verify the VPN credentials arrived" - if: matrix.chart == 'freesailarr' - env: - FREESAILARR_WG_KEY: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_KEY }} - FREESAILARR_WG_ADDRESSES: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_ADDRESSES }} - run: |- - : "${FREESAILARR_WG_KEY:?empty - check op://GitHub-Actions/freesailarr-mullvad-test-config/credential}" - : "${FREESAILARR_WG_ADDRESSES:?empty - check op://GitHub-Actions/freesailarr-mullvad-test-config/ip-address}" - echo "Both VPN values arrived non-empty." - - name: "Install k3d" - run: curl -sSfL https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG="$K3D_VERSION" bash - # One cluster per matrix leg, named after the chart. It used to be the - # fixed name "install-test", and that was wrong on GitHub too - it just - # could not show there, because GitHub gives every matrix leg its own VM - # with its own Docker daemon, so two legs never saw each other's - # cluster. The environment hid the bug for two years rather than - # preventing it. Gitea runs all legs of a matrix CONCURRENTLY in ONE job - # on ONE daemon, and there the fixed name collides at once: measured on - # the probe PR of #225, one leg logs "Re-using existing network - # 'k3d-install-test'", the other dies on "The container name - # /k3d-install-test-server-0 is already in use" - and then the first - # leg's rollback deletes the shared network out from under the second, - # which fails with "network k3d-install-test not found". Both legs red, - # neither cause in either message. DO NOT put the fixed name back. - # - # Why the bare chart name and not a prefixed one: k3d rejects a cluster - # name over 32 characters ("Cluster name must be <= 32 characters", - # measured against k3d v5.9.0 - 32 passes, 33 fails), and every - # container is named k3d--server-0 from it. The longest chart - # here is calibre-web-automated at 21, so "install-test-" would - # be 34 and break exactly that one chart and no other - the kind of - # prefix that is worse than the fixed name because it only sometimes - # breaks. The guard below states the limit instead of leaving it to be - # rediscovered. - - name: "Create k3d cluster" - run: | - CLUSTER="${{ matrix.chart }}" - if [ "${#CLUSTER}" -gt 32 ]; then - echo "::error::chart name '$CLUSTER' is ${#CLUSTER} characters and k3d allows at most 32 for a cluster name." - exit 1 - fi - - # Does THIS job run inside a container? That, and not which forge we - # are on, is what decides whether k3d's cluster is reachable (#227). - # - # k3d publishes the API server port on the Docker *host*. When the - # job runs straight on the host - GitHub's hosted runners - that is - # the same machine, and the "https://0.0.0.0:" k3d writes into - # the kubeconfig works. When the job itself is a container, as every - # Gitea Actions job is, the host is the dind container and 0.0.0.0 - # points at the job container itself, where nothing listens. The - # cluster comes up fine and the first kubectl call is refused. - # - # Deliberately NOT keyed on github.server_url, although that would - # pick the right branch today: a self-hosted GitHub runner in a - # container has exactly this problem, and a condition that says - # "GitHub" while meaning "not containerised" is the kind of lie this - # repo exists to remove. So the state is detected instead of - # inferred, and the detection is the same question the fix needs - # answered anyway: ask the daemon about ourselves. Inside a - # container that the daemon knows, this returns our network and k3d - # joins it, which puts k3s and this job on one bridge where Docker's - # own DNS resolves the container names. On a host it returns nothing, - # because the hostname is not a container, and the old path is taken - # unchanged. - # The "|| true" is load-bearing: this step runs under `bash -e -o - # pipefail`, and on a host `docker inspect` exits non-zero, which - # would abort the step instead of taking the else branch. - JOB_NET="$(docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}' "$(hostname)" 2>/dev/null | awk '{print $1}' || true)" - - if [ -n "$JOB_NET" ]; then - echo "Containerised job: $(hostname) on docker network '$JOB_NET' - k3d joins it." - k3d cluster create "$CLUSTER" --image "$K3S_IMAGE" --network "$JOB_NET" --wait --timeout 180s - # Same reason: the published port is useless from in here, so the - # kubeconfig is pointed at the load balancer container by name, - # over the shared network. If the API server certificate did not - # carry that name this would fail on the TLS handshake and the - # cluster-info below would say so - it holds without an extra - # --tls-san, measured on the probe PR of #225. - kubectl config set-cluster "k3d-$CLUSTER" --server="https://k3d-$CLUSTER-serverlb:6443" - else - echo "Job runs on the docker host - k3d's published port is reachable as written." - k3d cluster create "$CLUSTER" --image "$K3S_IMAGE" --wait --timeout 180s - fi - - # Reachability is proven HERE, and the shape of this check is the - # product of getting it wrong twice. Both mistakes are written down - # because both look like something to simplify away. - # - # It is a LOOP and not a single call. k3d's --wait returns when the - # node container reports ready, but the API server can still answer - # discovery with 503 for a moment after that. Measured: a single - # `kubectl cluster-info` right here failed with "ServiceUnavailable: - # the server is currently unable to handle the request" on one of - # two legs of the SAME GitHub run while the other passed. A check - # that is itself racy produces exactly the sporadic red this job - # must never have (#227) - and a test people have learned to re-run - # is one they have stopped believing. - # - # It counts READY NODES and not "did kubectl answer". `kubectl get - # nodes` exits 0 on an EMPTY list, so the first version passed - # against a cluster whose API server was up and whose node had never - # registered: the log said "No resources found", the step went - # green, and the failure surfaced five minutes later as a pod stuck - # in Pending with no node and no scheduling event - nowhere near its - # cause. A check that a broken cluster passes is worse than no - # check. `kubectl wait --for=condition=Ready nodes --all` is not the - # shorter way to write this: it has the identical hole, because with - # zero nodes the condition is vacuously satisfied. - # - # The evidence that this version earns its keep: only after it - # counted nodes did the real cause become visible in the dump below - # - k3s could not start containerd ("error creating fsnotify - # watcher: too many open files", an inotify limit on the host), and - # nothing above this line had ever said so. - for ATTEMPT in $(seq 1 90); do - READY_NODES="$(kubectl get nodes --no-headers 2>/dev/null | awk '$2 == "Ready"' | wc -l | tr -d ' ')" - [ "$READY_NODES" -ge 1 ] && break - if [ "$ATTEMPT" -eq 90 ]; then - echo "::error::the k3d cluster '$CLUSTER' answers but has no Ready node after 180s" - kubectl get nodes -o wide || true - kubectl describe nodes || true - echo "--- k3s server log ---" - docker logs "k3d-$CLUSTER-server-0" 2>&1 | tail -80 || true - exit 1 - fi - sleep 2 - done - kubectl get nodes -o wide - # Every chart reaching this step is one the exclusion filter let through, - # so the script's SKIP branch is never taken here - it exists for local - # runs. The variables below are empty for every chart but the one that - # declares them in ci//settings.env (REQUIRED_ENV); the script - # fails loudly if a declared one is missing, it never skips over it. - - name: "Install chart and wait for readiness" - env: - FREESAILARR_WG_KEY: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_KEY }} - FREESAILARR_WG_ADDRESSES: ${{ steps.load-freesailarr-vpn.outputs.FREESAILARR_WG_ADDRESSES }} - run: ci/run-install-test.sh "${{ matrix.chart }}" - # Must carry the same per-leg name as the create step above: a blanket - # cleanup, or the old fixed name, would tear down the neighbouring leg's - # cluster on a runner that shares one Docker daemon - which is precisely - # the half of the collision that turned one red leg into two. - - name: "Delete k3d cluster" - if: always() - run: k3d cluster delete "${{ matrix.chart }}" || true + # "install-charts", not "charts": the changed charts minus the ones in + # ci/excluded-charts.txt, filtered in detect-changed-charts, which also + # names every excluded chart of this pull request with its documented + # reason in its job summary (issue #231). The called workflow's matrix + # comment carries the rest of that lesson, including why the decision + # cannot be made per matrix leg instead. + uses: ./.github/workflows/install-test.yml + with: + charts: ${{ needs.detect-changed-charts.outputs.install-charts }} + # As the 24 publish-.yml hulls do, and measured to work on this + # instance (README, "Where this repo lives"). The called workflow declares + # exactly one secret, OP_CONNECT_TOKEN, and only its freesailarr leg reads + # it. + secrets: inherit diff --git a/README.md b/README.md index 1734248..62ed9d3 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,9 @@ Every chart has a wiki page of its own — `https://git.jk-dev.de/jlab-cluster/h Note: the registry **accepts a re-upload of an already published version** (HTTP 201) and replaces the stored artifact, so a republish of an existing version is still silent — see #160. - `change-app-version.yml` — manually dispatched app-version bump; the chart must be listed in its `chart-name` choice list. - `ci/helm-dependencies.sh ` — not a workflow but the step they all share: fetches a chart's declared dependencies into `charts/`, and returns silently for the charts that declare none (all but one). It runs in **five** places, because every helm command this repo builds on exits 1 while a declared dependency is missing: before lint and package in `validate-charts.yml`, before package in `publish-helm-chart.yml`, and inside `ci/run-install-test.sh`, `ci/run-quantity-render-test.sh` and `ci/run-trivy-config.sh`. See "Chart dependencies" in the style guidelines for what that costs. -- `validate-charts.yml` — PR pipeline: detects which charts a PR touches and, for exactly those, runs `helm lint --strict`, `helm package`, and a chart-version bump check (`template-chart` is exempt from the version check). PRs without chart changes skip the build matrix. After the build stage, an **install-test** job (the same changed-charts matrix, minus the charts listed in `ci/excluded-charts.txt`) installs each changed chart into an ephemeral [k3d](https://k3d.io) cluster on the runner and requires every workload to become Ready — see below. Both matrices follow the changed chart directories, so a PR that only changes the harness under `ci/` exercises nothing by itself; the Trivy scan has a fixed-sample fallback for that case, the install-test has none — bump the affected chart's patch version to pull it into the matrix. The **resource-quantity render test** is the exception to that rule: it runs over every chart on every PR — see below. +- `validate-charts.yml` — PR pipeline: detects which charts a PR touches and, for exactly those, runs `helm lint --strict`, `helm package`, and a chart-version bump check (`template-chart` is exempt from the version check). PRs without chart changes skip the build matrix. After the build stage, an **install-test** job (the same changed-charts matrix, minus the charts listed in `ci/excluded-charts.txt`) installs each changed chart into an ephemeral [k3d](https://k3d.io) cluster on the runner and requires every workload to become Ready — see below. Both matrices follow the changed chart directories, so a PR that only changes the harness under `ci/` exercises nothing by itself; the Trivy scan has a fixed-sample fallback for that case, the install-test has none — bump the affected chart's patch version to pull it into the matrix, or run the all-charts install-test after the merge. The **resource-quantity render test** is the exception to that rule: it runs over every chart on every PR — see below. +- `install-test.yml` — reusable workflow (`workflow_call`) holding the install-test **job** itself: the runner-label expression, the pinned `kubectl` with its skew check, `envsubst`, the containerised-job detection, the per-leg k3d cluster and the `freesailarr` VPN credentials. Its one input is a JSON array of chart names; its callers decide *which* charts are tested and whether the job runs at all, never *how*. Two callers: `validate-charts.yml` and `install-test-all-charts.yml`. **It is not copied for the second caller** — a second copy is a second chance to test differently, unnoticed, the same argument that gives `ci/excluded-charts.txt` exactly one parser. +- `install-test-all-charts.yml` — manually dispatched **all-charts install-test**: the same job, with a matrix over every installable chart instead of the changed ones. Optional input for a subset of charts (empty = all). See below for when to run it. ### Where this repo lives, and what a Gitea runner does differently @@ -69,7 +71,7 @@ Everything below was measured on this instance, not taken from documentation, an **Runner labels mean different machines than on GitHub.** `ubuntu-latest` here has **no Docker socket**; `ubuntu-docker` has one. Exactly one job cares — the install-test, because k3d is a Docker client — and its `runs-on` is therefore an expression rather than a fixed label, so the same file still works on a GitHub runner where `ubuntu-docker` does not exist. Every other job (lint/package, the quantity render, the Trivy scan) needs no socket and stays on `ubuntu-latest`. -**A matrix runs concurrently inside ONE job, under ONE status context.** This is the single most consequential difference and it has two halves. Operationally: all legs share one Docker daemon, which is what makes a per-leg resource name mandatory (see the cluster name below). For branch protection: **a required check per chart is not possible**, because there is only one context for the whole matrix — and its name carries the *unexpanded* matrix expression, e.g. `Validate Helm Charts / Install-test ${{ fromJSON(needs.detect-changed-charts.outputs.install-charts) }} (pull_request)`. Required-check names have to be read off a real run; they cannot be copied from GitHub. What *is* required on `main` today is **`validate-charts`**, as the single context it reports under, alongside a **review**: per chart is the part that is impossible, not required checks as such. Both are what makes Renovate's `platformAutomerge: true` safe — see "PR policy". +**A matrix runs concurrently inside ONE job, under ONE status context.** This is the single most consequential difference and it has two halves. Operationally: all legs share one Docker daemon, which is what makes a per-leg resource name mandatory (see the cluster name below). For branch protection: **a required check per chart is not possible**, because there is only one context for the whole matrix — and its name carries the *unexpanded* matrix expression, e.g. `Validate Helm Charts / Install-test ${{ fromJSON(needs.detect-changed-charts.outputs.install-charts) }} (pull_request)`. Required-check names have to be read off a real run; they cannot be copied from GitHub. That example name is from before the install-test job moved into the reusable `install-test.yml` (Gitea #52) — the matrix expression now reads `${{ fromJSON(inputs.charts) }}` and the context may carry the calling job's name as well, so **read the current name off the next chart PR before using it anywhere**. What *is* required on `main` today is **`validate-charts`**, as the single context it reports under, alongside a **review**: per chart is the part that is impossible, not required checks as such. Both are what makes Renovate's `platformAutomerge: true` safe — see "PR policy". **Expected runtimes**, so that a later deviation is visible as one: @@ -78,6 +80,7 @@ Everything below was measured on this instance, not taken from documentation, an | `Create k3d cluster` | 19–25 s | 22–23 s | | `install-test` job, two charts in the matrix (cyberchef + freesailarr) | 210–216 s over three runs, spread 6 s | ~145 s wall, two parallel VMs | | `install-test` job, freesailarr alone, ten containers and a real tunnel | 225 s (measured on the MediathekArr change, Gitea #20) | not comparable — Actions there were already off | +| `install-test` job, **20 charts** in the matrix — the accidental all-charts run of Gitea PR #48 | **12m41s**, one job, all legs concurrently on one Docker daemon; 19 of 20 charts Ready, the twentieth is Gitea #51 | never ran there | The runner is **not slower** — cluster creation is a touch faster. The difference in total time is **structural, not performance**: two matrix legs share one runner here, while GitHub gives each leg its own VM. Reading it as "Gitea is slower" leads to the wrong conclusion. @@ -130,6 +133,8 @@ Two details worth knowing before touching it: For every chart a PR changes, `ci/run-install-test.sh ` runs against a throwaway k3d cluster: it applies the stub `OnePasswordItem` CRD plus the chart's fixtures, `helm install`s the chart with its CI values, and waits for all Deployments/StatefulSets to roll out (per-chart timeout). Since probes and security hardening are active on all workloads, "Ready" means the app actually answers. On failure the job dumps pod descriptions, events and container logs. +The job itself is the reusable workflow **`install-test.yml`**; `validate-charts.yml` calls it with the changed charts, `install-test-all-charts.yml` with all of them. Everything about *how* a chart is installed is in that one file, deliberately — see "All-charts install-test" below. + Fixtures live at the **repo root** under `ci/` (deliberately *not* inside the chart directories, so they are neither packaged into the chart tgz nor trigger the publish workflows on merge): - `ci/common/` — fixtures applied for every chart (schema-less stub CRD for `OnePasswordItem`) @@ -145,6 +150,20 @@ Fixtures live at the **repo root** under `ci/` (deliberately *not* inside the ch Fixture images avoid Docker Hub where possible (`public.ecr.aws`/`ghcr.io`) to dodge runner rate limits. The job **blocks the PR** on failure — the stabilization phase of #22 ended when Renovate started automerging patch/minor updates, since a chart that no longer starts must not reach `main` and publish unattended. When adding a new chart, add its `ci//` fixtures (or an entry in `ci/excluded-charts.txt`) — the install-test fails if both are missing. The script also runs locally against any existing cluster: `k3d cluster create t && ci/run-install-test.sh && k3d cluster delete t`. +### All-charts install-test (on demand) + +`install-test-all-charts.yml` runs **the same job** as the PR pipeline — the same `install-test.yml`, the same pinned `kubectl`, the same fixtures — with a matrix over **every installable chart** instead of the charts a PR changed. `workflow_dispatch` only, **no schedule**: a daily run would spend compute on a question that changes rarely, and a red run nobody asked for is a red run nobody reads. + +It answers a different question from the PR pipeline. The PR install-test guards against what a change breaks; this one asks whether the environment under an **unchanged** chart still holds. The install-test matrix follows the charts a PR touches, so a chart nobody has edited since the forge move carries a green history from a *different runner* — and that green says nothing about today. **Run it after a change to the runner, to `K3S_IMAGE`, to the pinned `KUBECTL_VERSION`, to the fixtures under `ci/` — and after a forge move.** It is also the answer for a PR that only changes the harness and therefore tests nothing by itself: run it once the change is on `main`. + +Why it exists at all: the answer arrived once by accident. Gitea PR #48 corrected a *comment* in 21 charts, which made all of them "changed" and turned the run into the first all-charts install-test after the move — **19 of 20 matrix charts came up in 12m41s** in one job, and exactly one did not (`rss-bridge`, whose last green run was from a GitHub runner; Gitea #51). Worth having, and nobody had planned it. + +- **Input `charts`** — space- or comma-separated chart names, empty means all. That is what lets a single chart be tested against the runner, which is what Gitea #51 needs. A name that is not a chart directory, or one listed in `ci/excluded-charts.txt`, is a **loud error**: someone typed it, and a green run over zero charts would be the worst possible answer. An empty matrix fails the run for the same reason. +- **One throwaway cluster per chart, no reuse.** Reuse would save time and change what the test proves — leftovers from the previous install, a warm image cache, fixtures already applied. "Came up from nothing" is the statement. +- **Exclusions come from the one parser**, `ci/run-install-test.sh --list-excluded`, and the excluded charts appear in the job summary **with their justification**, exactly as in the PR pipeline. +- **The result is written down.** Every matrix leg records its own outcome next to the four facts that decide what the outcome means — date, runner label, `K3S_IMAGE`, `kubectl` version — and a final job records the verdict over the whole matrix. A green run nobody notes is unknown again next time. Per leg and not as one table, because a matrix leg only ever sees its own job summary. +- **Expected runtime: 20 charts in 12m41s** (Gitea PR #48, one job, all legs concurrently on one Docker daemon). The figure is there so a later deviation is visible as one; compare it against the chart count the run prints in its own summary — 24 at the time of writing, and one more with every new chart. The job timeout is 30 minutes per leg and lives in `install-test.yml`, i.e. it is the same for both callers on purpose. + ## Dependency updates (Renovate) Renovate runs as the shared `jlab-renovate-bot` instance on the Gitea side, which also serves the other repositories there — see "Renovate on Gitea" below. **There is no Renovate workflow in this repo, and none should be added.** (The `jlab-renovate` GitHub App that used to do this is switched off along with the rest of GitHub Actions.) What updates is unaffected either way: the config is **`renovate.json5`** in the repo root (JSON5 for the inline reasoning; requires Renovate >= 41 for `managerFilePatterns`/`customType`). There must be exactly one config file — do not add a second `renovate.json` or `.github/renovate.json`. diff --git a/ci/freesailarr/fixtures.env.yaml b/ci/freesailarr/fixtures.env.yaml index b170ba1..6f00f14 100644 --- a/ci/freesailarr/fixtures.env.yaml +++ b/ci/freesailarr/fixtures.env.yaml @@ -5,7 +5,7 @@ # (REQUIRED_ENV), and pipes the result straight into kubectl. # # The values come from a 1Password item holding a CI-only Mullvad key, loaded in -# the install-test job of .github/workflows/validate-charts.yml. A CI-only key +# the install-test job of .github/workflows/install-test.yml. A CI-only key # on purpose: Mullvad ties an address to a key and sends the tunnel to the # endpoint it saw last, so a shared key would let every pull request pull the # tunnel out from under the running stack. diff --git a/ci/run-install-test.sh b/ci/run-install-test.sh index 8609d36..e355a97 100755 --- a/ci/run-install-test.sh +++ b/ci/run-install-test.sh @@ -21,9 +21,11 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CI_DIR="$ROOT/ci" # --- The exclusion list, parsed in exactly one place ------------------------ -# Two consumers read ci/excluded-charts.txt: this script, and the install-test -# matrix in .github/workflows/validate-charts.yml, which drops the excluded -# charts before the matrix is built (issue #231). Two parsers would be two +# The consumers of ci/excluded-charts.txt are this script and the two jobs that +# build an install-test matrix - detect-changed-charts in +# .github/workflows/validate-charts.yml and select-charts in +# .github/workflows/install-test-all-charts.yml - both of which drop the +# excluded charts before the matrix exists (issue #231). Two parsers would be two # chances to select different charts - and the failure would be silent in the # worst direction, a chart the workflow believes excluded and the script # believes testable, or the reverse. So there is one parser, here, and the @@ -47,9 +49,10 @@ excluded_reason() { list_excluded | awk -F'\t' -v chart="$1" '$1 == chart { print $2; exit }' } -# Emits "" for every excluded chart. Read by the detect job -# of validate-charts.yml before any chart is picked, which is why this runs -# before the argument is required. +# Emits "" for every excluded chart. Read by both jobs that +# assemble an install-test matrix (detect-changed-charts in validate-charts.yml, +# select-charts in install-test-all-charts.yml) before any chart is picked, +# which is why this runs before the argument is required. if [ "${1:-}" = "--list-excluded" ]; then list_excluded exit 0 @@ -100,7 +103,7 @@ fi # REQUIRED_ENV="FREESAILARR_WG_KEY FREESAILARR_WG_ADDRESSES" # # The values come from the environment - in CI from 1Password Connect, see the -# install-test job in .github/workflows/validate-charts.yml. +# install-test job in .github/workflows/install-test.yml. # # A missing or empty value is a hard error on purpose. Skipping the chart # instead would restore exactly the green-but-meaningless signal this mechanism