# The all-charts install-test: the same job the pull-request pipeline runs, with # a matrix over every installable chart instead of the changed ones (Gitea #52). # # The question it answers is not "did this change break a chart" - that is the # pull-request pipeline's job - but "does the environment under an UNCHANGED # chart still hold". The install-test matrix follows the charts a pull request # touches, so a chart nobody has edited since the forge move carries a green # history from a different runner, and that green says nothing about today. # # It exists because the answer arrived once by accident: Gitea PR #48 corrected # a comment in 21 charts, which made every one of them "changed", and so became # the first all-charts run after the move - 19 of 20 matrix charts came up in # 12m41s in one job, and exactly one did not (rss-bridge, Gitea #51). That # result was worth having and nobody had planned it. This workflow makes it # repeatable. # # WHEN TO RUN IT: after a change to the runner, to K3S_IMAGE, to the pinned # KUBECTL_VERSION, to the fixtures under ci/ - and after a forge move. Not on a # schedule: it would spend compute every day on a question that changes rarely, # and a red run nobody asked for is a red run nobody reads. name: Install-test (all charts) on: workflow_dispatch: inputs: charts: required: false type: string default: "" description: "charts to test, separated by spaces or commas - empty means every installable chart" jobs: select-charts: name: "Select charts" runs-on: ubuntu-latest outputs: charts: ${{ steps.select.outputs.charts }} count: ${{ steps.select.outputs.count }} steps: - name: "Checkout Repository" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: "Select the charts to test" id: select env: # The dispatch input is read as an environment variable and never # interpolated into the script body. A "${{ inputs.charts }}" inside # the shell would let whoever triggers the run write commands into # this step; it is a small door and there is no reason to leave it # open. The name check further down is the second half of that: these # values end up as a matrix entry, a k3d cluster name and an argument # to ci/run-install-test.sh. REQUESTED: ${{ inputs.charts }} run: | FAIL=0 # --- The candidates ------------------------------------------------- # Empty input: every directory with a Chart.yaml, which is the same # definition detect-changed-charts uses for "this is a chart". No list # to maintain - adding a chart enrols it, the way the quantity render # test enrols one. # # A subset was asked for: exactly those names. This is what lets # someone test a single chart against the runner, which is what Gitea # #51 needs - and it is why an unknown or excluded name below is an # error rather than a silent omission. Asking for one chart and # getting a green run over zero charts would be the worst answer this # workflow could give. if [ -z "$(printf '%s' "$REQUESTED" | tr -d '[:space:],')" ]; then SUBSET=false CANDIDATES="$(ls -d -- */Chart.yaml 2>/dev/null | sed 's|/Chart.yaml$||' | sort)" echo "No subset requested - every chart directory is a candidate." else SUBSET=true CANDIDATES="$(printf '%s' "$REQUESTED" | tr ',' ' ' | tr -s '[:space:]' '\n' | grep -v '^$' | sort -u)" echo "Requested subset: $(echo "$CANDIDATES" | tr '\n' ' ')" fi for C in $CANDIDATES; do case "$C" in *[!a-z0-9-]*) echo "::error::'$C' is not a chart name - chart directories in this repository are lowercase letters, digits and dashes." FAIL=1 continue ;; esac if [ ! -f "$C/Chart.yaml" ]; then echo "::error::'$C' is not a chart directory of this repository (no $C/Chart.yaml)." FAIL=1 fi done [ "$FAIL" -eq 0 ] || exit 1 CHARTS="$(printf '%s\n' $CANDIDATES | jq -R . | jq -sc 'unique')" # --- The exclusion list, through the one parser --------------------- # ci/run-install-test.sh --list-excluded emits "" # and is the only thing in this repository that reads the format of # ci/excluded-charts.txt - here no more than in validate-charts.yml, # because a second parser would be a second chance to select # different charts, silently and in either direction (issue #231). # The comparison is jq's "!=" on whole strings, so a chart name is # never read as a pattern. # # The two callers differ in what they do with a hit, and on purpose. # Over all charts, an excluded chart is dropped and named with its # reason, exactly as in the pull-request pipeline. In a requested # subset it is an ERROR: someone typed that name, and quietly # returning a green run for a chart that was never attempted is the # lie #231 removed from the check list. SUMMARY_ROWS="" while IFS=$'\t' read -r EX_NAME EX_REASON; do [ -n "$EX_NAME" ] || continue if [ "$(echo "$CHARTS" | jq -r --arg c "$EX_NAME" 'index($c) != null')" = "true" ]; then if [ "$SUBSET" = true ]; then echo "::error::chart '$EX_NAME' was requested but is listed in ci/excluded-charts.txt: $EX_REASON" FAIL=1 fi # A "|" in the justification would break out of the table cell. SUMMARY_ROWS="${SUMMARY_ROWS}| \`$EX_NAME\` | ${EX_REASON//|/\\|} |"$'\n' echo "Excluded from the install-test: $EX_NAME - $EX_REASON" fi CHARTS="$(echo "$CHARTS" | jq -c --arg c "$EX_NAME" 'map(select(. != $c))')" done <<< "$(ci/run-install-test.sh --list-excluded)" [ "$FAIL" -eq 0 ] || exit 1 # An empty matrix does not fail - the install-test job would simply # not be created and the run would end green, having tested nothing. # That is the shape of signal this whole issue is about, so it is a # loud error instead. if [ "$CHARTS" = "[]" ]; then echo "::error::no chart left to test - every candidate is listed in ci/excluded-charts.txt." exit 1 fi COUNT="$(echo "$CHARTS" | jq -r 'length')" echo "charts=$CHARTS" >> "$GITHUB_OUTPUT" echo "count=$COUNT" >> "$GITHUB_OUTPUT" echo "Install-test matrix ($COUNT charts): $CHARTS" # --- The run record, first half ------------------------------------- # What was asked for, when, and against what. The per-chart outcome # and the environment each leg measured are written by the legs # themselves (the called workflow's "Record the result" step), the # verdict by the record job below. A green run nobody notes is # unknown again next time. { echo "## All-charts install-test" echo echo "- Started: **$(date -u '+%Y-%m-%d %H:%M') UTC**" echo "- Requested: $( [ "$SUBSET" = true ] && echo "subset \`$REQUESTED\`" || echo "**every installable chart**" )" echo "- Matrix: **$COUNT charts** — $(echo "$CHARTS" | jq -r 'map("`" + . + "`") | join(", ")')" echo "- Expected runtime, so a deviation is visible as one: **20 charts in 12m41s**, measured on Gitea PR #48 — one job, all legs concurrently on one Docker daemon. The job timeout is 30 minutes per leg." echo "- One throwaway k3d cluster per chart, no reuse: a reused cluster would carry leftovers, a warm image cache and fixtures already applied, and \"came up from nothing\" is the statement this test exists to make." echo echo "Each chart's outcome, with the runner label, \`K3S_IMAGE\` and \`kubectl\` version it was measured against, is in the install-test job summaries; the verdict over the whole matrix is at the end." echo } >> "$GITHUB_STEP_SUMMARY" if [ -n "$SUMMARY_ROWS" ]; then { echo "### Charts excluded from the install-test" echo echo "These charts are listed in \`ci/excluded-charts.txt\` and get **no** install-test job. There is no green tick for them, because nothing was attempted (issue #231)." echo echo "| Chart | Documented reason |" echo "| --- | --- |" printf '%s' "$SUMMARY_ROWS" echo } >> "$GITHUB_STEP_SUMMARY" fi # The same workflow validate-charts.yml calls, with the same inputs, differing # only in the chart list. Not a copy of the job: see the header of # .github/workflows/install-test.yml for what would drift. all-charts: name: "All charts" needs: select-charts uses: ./.github/workflows/install-test.yml with: charts: ${{ needs.select-charts.outputs.charts }} secrets: inherit # The verdict over the matrix as a whole. A matrix leg only ever sees its own # job summary, so this is the one place that can say "all of them" - and the # aggregate is what the question "has the environment changed" is answered # with. It runs on failure too, hence "if: always()". record: name: "Record the run" needs: [select-charts, all-charts] if: always() runs-on: ubuntu-latest steps: - name: "Write the verdict" env: RESULT: ${{ needs.all-charts.result }} COUNT: ${{ needs.select-charts.outputs.count }} run: | case "$RESULT" in success) VERDICT="**all $COUNT charts installed and became Ready** on this runner, on $(date -u '+%Y-%m-%d')" ;; failure) VERDICT="**at least one chart did not come up.** Which ones, and whether the chart failed or the cluster never came up, is in the per-chart records above. That is a result and not an accident to re-run: rss-bridge failing this way on Gitea PR #48 is why Gitea #51 exists" ;; skipped|cancelled) VERDICT="**no verdict** - the matrix did not run ($RESULT). Nothing was tested; do not read this run as a green one" ;; *) VERDICT="**no verdict** - unexpected matrix result '$RESULT'" ;; esac { echo "## Verdict" echo echo "$VERDICT" echo echo "Worth writing down where it belongs: a result from this run is what the next \"has anything drifted\" question is compared against. If it moved a chart forward or backward, say so on the chart's wiki page or in \`Lessons-Learned\`." } >> "$GITHUB_STEP_SUMMARY"