217 lines
11 KiB
YAML
217 lines
11 KiB
YAML
# The all-charts install-test: the same job the pull-request pipeline runs, with
|
|
# a matrix over every installable chart instead of the changed ones (Gitea #52).
|
|
#
|
|
# The question it answers is not "did this change break a chart" - that is the
|
|
# pull-request pipeline's job - but "does the environment under an UNCHANGED
|
|
# chart still hold". The install-test matrix follows the charts a pull request
|
|
# touches, so a chart nobody has edited since the forge move carries a green
|
|
# history from a different runner, and that green says nothing about today.
|
|
#
|
|
# It exists because the answer arrived once by accident: Gitea PR #48 corrected
|
|
# a comment in 21 charts, which made every one of them "changed", and so became
|
|
# the first all-charts run after the move - 19 of 20 matrix charts came up in
|
|
# 12m41s in one job, and exactly one did not (rss-bridge, Gitea #51). That
|
|
# result was worth having and nobody had planned it. This workflow makes it
|
|
# repeatable.
|
|
#
|
|
# WHEN TO RUN IT: after a change to the runner, to K3S_IMAGE, to the pinned
|
|
# KUBECTL_VERSION, to the fixtures under ci/ - and after a forge move. Not on a
|
|
# schedule: it would spend compute every day on a question that changes rarely,
|
|
# and a red run nobody asked for is a red run nobody reads.
|
|
name: Install-test (all charts)
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
charts:
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
description: "charts to test, separated by spaces or commas - empty means every installable chart"
|
|
jobs:
|
|
select-charts:
|
|
name: "Select charts"
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
charts: ${{ steps.select.outputs.charts }}
|
|
count: ${{ steps.select.outputs.count }}
|
|
steps:
|
|
- name: "Checkout Repository"
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
- name: "Select the charts to test"
|
|
id: select
|
|
env:
|
|
# The dispatch input is read as an environment variable and never
|
|
# interpolated into the script body. A "${{ inputs.charts }}" inside
|
|
# the shell would let whoever triggers the run write commands into
|
|
# this step; it is a small door and there is no reason to leave it
|
|
# open. The name check further down is the second half of that: these
|
|
# values end up as a matrix entry, a k3d cluster name and an argument
|
|
# to ci/run-install-test.sh.
|
|
REQUESTED: ${{ inputs.charts }}
|
|
run: |
|
|
FAIL=0
|
|
|
|
# --- The candidates -------------------------------------------------
|
|
# Empty input: every directory with a Chart.yaml, which is the same
|
|
# definition detect-changed-charts uses for "this is a chart". No list
|
|
# to maintain - adding a chart enrols it, the way the quantity render
|
|
# test enrols one.
|
|
#
|
|
# A subset was asked for: exactly those names. This is what lets
|
|
# someone test a single chart against the runner, which is what Gitea
|
|
# #51 needs - and it is why an unknown or excluded name below is an
|
|
# error rather than a silent omission. Asking for one chart and
|
|
# getting a green run over zero charts would be the worst answer this
|
|
# workflow could give.
|
|
if [ -z "$(printf '%s' "$REQUESTED" | tr -d '[:space:],')" ]; then
|
|
SUBSET=false
|
|
CANDIDATES="$(ls -d -- */Chart.yaml 2>/dev/null | sed 's|/Chart.yaml$||' | sort)"
|
|
echo "No subset requested - every chart directory is a candidate."
|
|
else
|
|
SUBSET=true
|
|
CANDIDATES="$(printf '%s' "$REQUESTED" | tr ',' ' ' | tr -s '[:space:]' '\n' | grep -v '^$' | sort -u)"
|
|
echo "Requested subset: $(echo "$CANDIDATES" | tr '\n' ' ')"
|
|
fi
|
|
|
|
for C in $CANDIDATES; do
|
|
case "$C" in
|
|
*[!a-z0-9-]*)
|
|
echo "::error::'$C' is not a chart name - chart directories in this repository are lowercase letters, digits and dashes."
|
|
FAIL=1
|
|
continue
|
|
;;
|
|
esac
|
|
if [ ! -f "$C/Chart.yaml" ]; then
|
|
echo "::error::'$C' is not a chart directory of this repository (no $C/Chart.yaml)."
|
|
FAIL=1
|
|
fi
|
|
done
|
|
[ "$FAIL" -eq 0 ] || exit 1
|
|
|
|
CHARTS="$(printf '%s\n' $CANDIDATES | jq -R . | jq -sc 'unique')"
|
|
|
|
# --- The exclusion list, through the one parser ---------------------
|
|
# ci/run-install-test.sh --list-excluded emits "<chart><TAB><reason>"
|
|
# and is the only thing in this repository that reads the format of
|
|
# ci/excluded-charts.txt - here no more than in validate-charts.yml,
|
|
# because a second parser would be a second chance to select
|
|
# different charts, silently and in either direction (issue #231).
|
|
# The comparison is jq's "!=" on whole strings, so a chart name is
|
|
# never read as a pattern.
|
|
#
|
|
# The two callers differ in what they do with a hit, and on purpose.
|
|
# Over all charts, an excluded chart is dropped and named with its
|
|
# reason, exactly as in the pull-request pipeline. In a requested
|
|
# subset it is an ERROR: someone typed that name, and quietly
|
|
# returning a green run for a chart that was never attempted is the
|
|
# lie #231 removed from the check list.
|
|
SUMMARY_ROWS=""
|
|
while IFS=$'\t' read -r EX_NAME EX_REASON; do
|
|
[ -n "$EX_NAME" ] || continue
|
|
if [ "$(echo "$CHARTS" | jq -r --arg c "$EX_NAME" 'index($c) != null')" = "true" ]; then
|
|
if [ "$SUBSET" = true ]; then
|
|
echo "::error::chart '$EX_NAME' was requested but is listed in ci/excluded-charts.txt: $EX_REASON"
|
|
FAIL=1
|
|
fi
|
|
# A "|" in the justification would break out of the table cell.
|
|
SUMMARY_ROWS="${SUMMARY_ROWS}| \`$EX_NAME\` | ${EX_REASON//|/\\|} |"$'\n'
|
|
echo "Excluded from the install-test: $EX_NAME - $EX_REASON"
|
|
fi
|
|
CHARTS="$(echo "$CHARTS" | jq -c --arg c "$EX_NAME" 'map(select(. != $c))')"
|
|
done <<< "$(ci/run-install-test.sh --list-excluded)"
|
|
[ "$FAIL" -eq 0 ] || exit 1
|
|
|
|
# An empty matrix does not fail - the install-test job would simply
|
|
# not be created and the run would end green, having tested nothing.
|
|
# That is the shape of signal this whole issue is about, so it is a
|
|
# loud error instead.
|
|
if [ "$CHARTS" = "[]" ]; then
|
|
echo "::error::no chart left to test - every candidate is listed in ci/excluded-charts.txt."
|
|
exit 1
|
|
fi
|
|
|
|
COUNT="$(echo "$CHARTS" | jq -r 'length')"
|
|
echo "charts=$CHARTS" >> "$GITHUB_OUTPUT"
|
|
echo "count=$COUNT" >> "$GITHUB_OUTPUT"
|
|
echo "Install-test matrix ($COUNT charts): $CHARTS"
|
|
|
|
# --- The run record, first half -------------------------------------
|
|
# What was asked for, when, and against what. The per-chart outcome
|
|
# and the environment each leg measured are written by the legs
|
|
# themselves (the called workflow's "Record the result" step), the
|
|
# verdict by the record job below. A green run nobody notes is
|
|
# unknown again next time.
|
|
{
|
|
echo "## All-charts install-test"
|
|
echo
|
|
echo "- Started: **$(date -u '+%Y-%m-%d %H:%M') UTC**"
|
|
echo "- Requested: $( [ "$SUBSET" = true ] && echo "subset \`$REQUESTED\`" || echo "**every installable chart**" )"
|
|
echo "- Matrix: **$COUNT charts** — $(echo "$CHARTS" | jq -r 'map("`" + . + "`") | join(", ")')"
|
|
echo "- Expected runtime, so a deviation is visible as one: **20 charts in 12m41s**, measured on Gitea PR #48 — one job, all legs concurrently on one Docker daemon. The job timeout is 30 minutes per leg."
|
|
echo "- One throwaway k3d cluster per chart, no reuse: a reused cluster would carry leftovers, a warm image cache and fixtures already applied, and \"came up from nothing\" is the statement this test exists to make."
|
|
echo
|
|
echo "Each chart's outcome, with the runner label, \`K3S_IMAGE\` and \`kubectl\` version it was measured against, is in the install-test job summaries; the verdict over the whole matrix is at the end."
|
|
echo
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
if [ -n "$SUMMARY_ROWS" ]; then
|
|
{
|
|
echo "### Charts excluded from the install-test"
|
|
echo
|
|
echo "These charts are listed in \`ci/excluded-charts.txt\` and get **no** install-test job. There is no green tick for them, because nothing was attempted (issue #231)."
|
|
echo
|
|
echo "| Chart | Documented reason |"
|
|
echo "| --- | --- |"
|
|
printf '%s' "$SUMMARY_ROWS"
|
|
echo
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
# The same workflow validate-charts.yml calls, with the same inputs, differing
|
|
# only in the chart list. Not a copy of the job: see the header of
|
|
# .github/workflows/install-test.yml for what would drift.
|
|
all-charts:
|
|
name: "All charts"
|
|
needs: select-charts
|
|
uses: ./.github/workflows/install-test.yml
|
|
with:
|
|
charts: ${{ needs.select-charts.outputs.charts }}
|
|
secrets: inherit
|
|
|
|
# The verdict over the matrix as a whole. A matrix leg only ever sees its own
|
|
# job summary, so this is the one place that can say "all of them" - and the
|
|
# aggregate is what the question "has the environment changed" is answered
|
|
# with. It runs on failure too, hence "if: always()".
|
|
record:
|
|
name: "Record the run"
|
|
needs: [select-charts, all-charts]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: "Write the verdict"
|
|
env:
|
|
RESULT: ${{ needs.all-charts.result }}
|
|
COUNT: ${{ needs.select-charts.outputs.count }}
|
|
run: |
|
|
case "$RESULT" in
|
|
success)
|
|
VERDICT="**all $COUNT charts installed and became Ready** on this runner, on $(date -u '+%Y-%m-%d')"
|
|
;;
|
|
failure)
|
|
VERDICT="**at least one chart did not come up.** Which ones, and whether the chart failed or the cluster never came up, is in the per-chart records above. That is a result and not an accident to re-run: rss-bridge failing this way on Gitea PR #48 is why Gitea #51 exists"
|
|
;;
|
|
skipped|cancelled)
|
|
VERDICT="**no verdict** - the matrix did not run ($RESULT). Nothing was tested; do not read this run as a green one"
|
|
;;
|
|
*)
|
|
VERDICT="**no verdict** - unexpected matrix result '$RESULT'"
|
|
;;
|
|
esac
|
|
{
|
|
echo "## Verdict"
|
|
echo
|
|
echo "$VERDICT"
|
|
echo
|
|
echo "Worth writing down where it belongs: a result from this run is what the next \"has anything drifted\" question is compared against. If it moved a chart forward or backward, say so on the chart's wiki page or in \`Lessons-Learned\`."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|