#!/usr/bin/env bash # HTTPRoute render test: every HTTPRoute a chart renders is held against the # Services of the SAME render, and against the one condition without which the # route attaches to nothing. # # Usage: ci/run-httproute-match-test.sh [chart ...] (no argument = all charts) # # WHAT IT GUARDS AGAINST, AND WHY NEITHER A RENDER NOR AN INSTALL DOES (#114) # # An HTTPRoute is the Gateway-API alternative to the Ingress (README, "Ingress # and Gateway API"), and it has two silent failure modes that look exactly like # a working route from the outside: # # parentRefs EMPTY - a route without a parent attaches to no Gateway. The # object is created, it is schema-valid, no controller rejects it, and it # routes nothing. There is no event and no condition; the only symptom is a # hostname that does not answer. The charts abort the render over this, and # this test is the half that catches a chart which forgot to. # backendRefs pointing at NOTHING - the reference is by name, in the release # namespace, and nothing validates that the name exists. A typo, a renamed # Service, or a port number that no longer matches the Service's port # yields a route that resolves to no endpoint. ResolvedRefs goes False in # the route's STATUS, which only exists once a controller has looked at it # - so in CI, where no Gateway controller runs, nothing reports it at all. # # And the install-test cannot close either gap, even with the real CRDs in the # cluster: schema validation accepts an empty parentRefs list and accepts a # backendRef to a Service that does not exist. The CRDs prove the object is # WELL-FORMED; this test is what proves it POINTS SOMEWHERE. # # Over ALL charts and not the changed ones, for the same reason as # ci/run-quantity-render-test.sh and ci/run-selector-match-test.sh: the mistake # arrives by copy into a chart whose own diff nobody reads for it - and this # convention is going to be copied 20 more times when the sweep of #114 runs. # Running over everything also means there is no list to maintain: adding a # chart enrols it. # # HOW THE YAML IS PARSED # # By Helm, through ci/yaml2json, exactly as ci/run-selector-match-test.sh does # it - the runners have neither pyyaml nor yq, and a parser written in awk # would be able to read nothing, find no route and report no finding, leaving # this test green for a chart it never looked at. The only textual step is the # document split on a line that is exactly "---" in column one, and getting # that wrong is not a false pass: the fragment stops being valid YAML and helm # exits 1. See that script's header for the long form. # # WHAT IS DELIBERATELY NOT CHECKED, so the gaps are not mistaken for coverage # # - THAT TRAFFIC FLOWS. No Gateway controller runs in CI and none will; a # route's status is written by the implementation, so "Accepted: True" is # not obtainable here. What is obtainable: the object is schema-valid # (install-test, against the real CRDs) and it names a Service and a port # that exist (this test). Routing itself is unproven, by construction. # - MUTUAL EXCLUSION with the Ingress. Every chart aborts its own render when # ingress.enabled and httpRoute.enabled are both true, which is where that # check belongs: this test renders ONE configuration per chart, and a # render-level rule here would have to assume the switch is chart-global. # That holds for the pilot and may not hold for the multi-app charts of the # sweep (matrix-synapse renders in a loop, keycloak and bulwark have two # Ingresses each), so the rule is not written down here where it would have # to be undone. # - CROSS-NAMESPACE backendRefs and non-Service backends. A backendRef into # another namespace needs a ReferenceGrant on the other side and its target # is by definition not in this render; a backend that is not a Service is # some implementation's own kind. Both are REPORTED as "not checked" and # are never counted as passing, because a green tick for a code path nobody # entered is worse than an honest gap (README, "Two rules for every check # added here"). # - A RULE WITHOUT backendRefs. Legal - a rule carrying only a # RequestRedirect filter needs no backend - so it is reported, not failed. # # AND THE ANTI-VACUUM GUARDS, because this test can be green over nothing. The # convention ships with httpRoute.enabled: false, so a chart renders no route # at all unless its CI values or an overlay switch the mode on. Two rules: # # 1. ci/httproute/.yaml is an overlay applied last, whose only job is # to put a chart into Gateway mode for this test. A chart that HAS one and # still renders no HTTPRoute FAILS - the overlay exists for exactly that. # 2. If no HTTPRoute at all was checked across the whole run, the run fails. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CI_DIR="$ROOT/ci" YAML2JSON="$CI_DIR/yaml2json" OVERLAY_DIR="$CI_DIR/httproute" RELEASE="httproute" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT # --- Which charts ----------------------------------------------------------- CHARTS=() if [ "$#" -gt 0 ]; then CHARTS=("$@") else for DIR in "$ROOT"/*/; do [ -f "$DIR/Chart.yaml" ] || continue CHARTS+=("$(basename "$DIR")") done fi if [ "${#CHARTS[@]}" -eq 0 ]; then echo "ERROR: no chart directories found under $ROOT" >&2 exit 1 fi FAILED="" CHECKED="" NOT_EXERCISED="" WITH_GAPS="" TOTAL_ASSERTIONS=0 SUMMARY_ROWS="" for CHART in "${CHARTS[@]}"; do if [ ! -f "$ROOT/$CHART/Chart.yaml" ]; then echo "ERROR: '$CHART' is not a chart directory" >&2 exit 1 fi # Same reason as in the other two render tests: a chart with a declared # dependency cannot be rendered at all while charts/ is empty, and this job # runs over EVERY chart on EVERY pull request - so without this line a PR # touching nothing but a README would go red on the charts that have one. "$ROOT/ci/helm-dependencies.sh" "$ROOT/$CHART" # Values: the same precedence the other render tests use, so a chart's CI # values are maintained in one place and satisfy its required() values. VALUES_ARGS=() VALUES_LABEL="chart defaults" if [ -f "$CI_DIR/$CHART/test-values.yaml" ]; then VALUES_ARGS=(-f "$CI_DIR/$CHART/test-values.yaml") VALUES_LABEL="ci/$CHART/test-values.yaml" elif [ -f "$CI_DIR/trivy/values/$CHART.yaml" ]; then VALUES_ARGS=(-f "$CI_DIR/trivy/values/$CHART.yaml") VALUES_LABEL="ci/trivy/values/$CHART.yaml" fi HAS_OVERLAY="" if [ -f "$OVERLAY_DIR/$CHART.yaml" ]; then # Last, so it wins over the install-test values. VALUES_ARGS+=(-f "$OVERLAY_DIR/$CHART.yaml") VALUES_LABEL="$VALUES_LABEL + ci/httproute/$CHART.yaml" HAS_OVERLAY=1 fi echo "::group::$CHART ($VALUES_LABEL)" RENDER="$WORK/$CHART.manifest.yaml" if ! helm template "$RELEASE" "$ROOT/$CHART" \ ${VALUES_ARGS[@]+"${VALUES_ARGS[@]}"} >"$RENDER" 2>"$WORK/err"; then sed 's/^/ /' "$WORK/err" echo "::endgroup::" echo "::error title=httproute $CHART::$CHART does not render with its CI values, so no HTTPRoute could be checked. Reproduce: helm template $RELEASE $CHART $(printf '%s ' "${VALUES_ARGS[@]+${VALUES_ARGS[@]}}")" FAILED="$FAILED $CHART" continue fi # --- The document split, and the only textual step in this script ---------- # One YAML sequence item per rendered document, content indented by four and # the item marker on a line of its own - so a document whose first line is a # comment ("# Source: ...") needs no special case. A wrong split does not # produce a false pass: the result stops being parsable and helm exits 1. DOCS="$WORK/$CHART.docs.yaml" awk ' BEGIN { print "docs:"; started = 0 } /^---[ \t]*$/ { print " -"; started = 1; next } { if (!started) { print " -"; started = 1 } if (length($0) == 0) print ""; else print " " $0 } ' "$RENDER" >"$DOCS" JSON="$WORK/$CHART.json" if ! helm template y2j "$YAML2JSON" -f "$DOCS" 2>"$WORK/err" \ | awk '/^\{/ { print; exit }' >"$JSON"; then sed 's/^/ /' "$WORK/err" echo "::endgroup::" echo "::error title=httproute $CHART::the rendered manifests of $CHART could not be re-read as YAML by ci/yaml2json. That is a parse failure, not an HTTPRoute finding: either the render contains a document this script's '---' split cut in the wrong place, or the manifest is not valid YAML. Helm's own message is in the job log." FAILED="$FAILED $CHART" continue fi if [ ! -s "$JSON" ]; then echo "::endgroup::" echo "::error title=httproute $CHART::ci/yaml2json produced no JSON for $CHART - the manifests were not re-read, so nothing was checked." FAILED="$FAILED $CHART" continue fi # --- The comparison -------------------------------------------------------- # Inline and on the standard library only, as in the two sibling tests. It # writes one "VERDICT|detail" line per assertion and never decides the exit # code itself: the loop below does, so a verdict can never be lost to a # swallowed status. set +e python3 - "$CHART" "$JSON" >"$WORK/$CHART.out" 2>"$WORK/$CHART.pyerr" <<'PY' import json, sys chart, path = sys.argv[1], sys.argv[2] docs = json.load(open(path)).get("docs") or [] docs = [d for d in docs if isinstance(d, dict)] def name_of(doc): meta = doc.get("metadata") if isinstance(meta, dict) and meta.get("name"): return str(meta["name"]) return "" % doc.get("kind", "object") # Every Service of the render, as name -> the set of port NUMBERS it exposes. # Port numbers and not names on purpose: an HTTPRoute backendRef carries a # PortNumber and cannot reference a port by name, which is exactly where it # differs from the Ingress backend this convention replaces - and exactly why # a chart has to keep the number in the route in step with its Service by # hand. services = {} for doc in docs: if doc.get("kind") != "Service": continue ports = set() spec = doc.get("spec") if isinstance(doc.get("spec"), dict) else {} for port in (spec.get("ports") or []): if isinstance(port, dict) and port.get("port") is not None: ports.add(str(port["port"])) services[name_of(doc)] = ports routes = [d for d in docs if d.get("kind") == "HTTPRoute"] lines = [] for doc in routes: route = "HTTPRoute %s" % name_of(doc) spec = doc.get("spec") if isinstance(doc.get("spec"), dict) else {} # 1. parentRefs must be a non-empty list. The whole point: a route without # a parent is valid, created, and inert. parents = spec.get("parentRefs") if not isinstance(parents, list) or not parents: lines.append("FAIL|%s: %s has %s parentRefs. A route without a parent " "attaches to NO Gateway: the object is created, it is " "schema-valid, no controller rejects it and it routes " "nothing - no event, no condition, the only symptom is a " "hostname that does not answer. The chart must abort its " "own render on an empty httpRoute.parentRefs" % (chart, route, "no" if parents is None else "an empty list of")) else: bad = [p for p in parents if not isinstance(p, dict) or not str(p.get("name") or "").strip()] if bad: lines.append("FAIL|%s: %s has %d parentRef(s) without a usable " "'name' (%r). ParentReference.name is required by the " "API; a reference without it attaches to nothing" % (chart, route, len(bad), bad)) else: lines.append("OK|%s: %s has %d parentRef(s): %s" % (chart, route, len(parents), ", ".join("%s%s" % (p["name"], "/%s" % p["namespace"] if p.get("namespace") else "") for p in parents))) # 2. Every backendRefs entry, against the Services of the SAME render. rules = spec.get("rules") if not isinstance(rules, list) or not rules: lines.append("FAIL|%s: %s renders no rules at all, so it matches no " "request and forwards nothing" % (chart, route)) continue for i, rule in enumerate(rules): tag = "%s rules[%d]" % (route, i) if not isinstance(rule, dict): lines.append("FAIL|%s: %s is not a mapping (%r)" % (chart, tag, rule)) continue backends = rule.get("backendRefs") if not backends: # Legal: a rule carrying only a RequestRedirect or a # RequestMirror filter needs no backend. Reported, never counted. lines.append("SKIP|%s: %s has no backendRefs. That is legal for a " "filter-only rule (a redirect, for instance) and is " "not evaluated here - but a rule meant to forward " "traffic and carrying no backend answers 500 by " "specification" % (chart, tag)) continue if not isinstance(backends, list): lines.append("FAIL|%s: %s backendRefs is not a list (%r)" % (chart, tag, backends)) continue for j, backend in enumerate(backends): where = "%s.backendRefs[%d]" % (tag, j) if not isinstance(backend, dict): lines.append("FAIL|%s: %s is not a mapping (%r)" % (chart, where, backend)) continue # kind and group default to Service and the core group. kind = str(backend.get("kind") or "Service") group = str(backend.get("group") or "") if kind != "Service" or group not in ("", "core"): lines.append("SKIP|%s: %s references %s.%s, which is not a core " "Service - some implementation's own backend kind. " "This test cannot resolve it and does not count it " "as passing" % (chart, where, group or "core", kind)) continue namespace = backend.get("namespace") if namespace: lines.append("SKIP|%s: %s references Service %s in namespace %s. A " "cross-namespace backend needs a ReferenceGrant on " "the other side and its target is by definition not " "in this render, so it is reported rather than " "checked" % (chart, where, backend.get("name"), namespace)) continue target = str(backend.get("name") or "") if not target: lines.append("FAIL|%s: %s has no 'name' - it references no " "backend at all" % (chart, where)) continue if target not in services: lines.append("FAIL|%s: %s references Service %r, which the SAME " "render does not create. A backendRef is resolved " "by name at runtime and validated by nothing: the " "route is accepted, ResolvedRefs goes False in a " "status no controller writes in CI, and the " "hostname answers with the Gateway's error instead " "of the app. Services this render does create: %s" % (chart, where, target, ", ".join(sorted(services)) or "none at all")) continue port = backend.get("port") if port is None: lines.append("FAIL|%s: %s references Service %r without a port. " "A backendRef to a Service must carry one - the " "CRD schema cannot require it conditionally, so an " "omitted port is accepted at apply time and " "resolves to nothing. The Service exposes: %s" % (chart, where, target, ", ".join(sorted(services[target])) or "no ports")) continue if str(port) not in services[target]: lines.append("FAIL|%s: %s references Service %r on port %s, which " "that Service does not expose (it serves %s). " "backendRefs takes a port NUMBER and cannot name a " "port, so this pair is kept in step by hand and " "drifts the moment one side is edited alone" % (chart, where, target, port, ", ".join(sorted(services[target])) or "no ports")) continue lines.append("OK|%s: %s resolves to Service %s:%s, created by the same " "render" % (chart, where, target, port)) for line in lines: print(line) PY RC=$? set -e if [ "$RC" -gt 1 ]; then sed 's/^/ /' "$WORK/$CHART.pyerr" echo "::endgroup::" echo "::error title=httproute $CHART::the HTTPRoute comparison itself failed for $CHART (python exit $RC). Treated as a failure, never as a pass." FAILED="$FAILED $CHART" continue fi CHART_FAIL="" CHART_OK=0 CHART_SKIP=0 while IFS='|' read -r VERDICT DETAIL; do case "$VERDICT" in OK) CHART_OK=$((CHART_OK + 1)) echo " ok $DETAIL" ;; SKIP) CHART_SKIP=$((CHART_SKIP + 1)) echo " gap $DETAIL" ;; FAIL) CHART_FAIL=1 echo " FAIL $DETAIL" echo "::error title=httproute $CHART::$DETAIL" ;; *) [ -n "$VERDICT" ] && echo " $VERDICT $DETAIL" ;; esac done <"$WORK/$CHART.out" echo "::endgroup::" TOTAL_ASSERTIONS=$((TOTAL_ASSERTIONS + CHART_OK)) if [ -n "$CHART_FAIL" ]; then FAILED="$FAILED $CHART" SUMMARY_ROWS="${SUMMARY_ROWS}| \`$CHART\` | **failed** | $CHART_OK | $CHART_SKIP |"$'\n' echo "$CHART: FAILED" elif [ "$CHART_OK" -gt 0 ]; then CHECKED="$CHECKED $CHART" [ "$CHART_SKIP" -gt 0 ] && WITH_GAPS="$WITH_GAPS $CHART" SUMMARY_ROWS="${SUMMARY_ROWS}| \`$CHART\` | ok | $CHART_OK | $CHART_SKIP |"$'\n' echo "$CHART: ok ($CHART_OK assertion(s) held, $CHART_SKIP not checked)" elif [ -n "$HAS_OVERLAY" ]; then # The overlay's ONE job is to put this chart into Gateway mode. If no # HTTPRoute came out, this test is vacuum-green for the chart the overlay # was written for, which is the failure the overlay prevents. FAILED="$FAILED $CHART" SUMMARY_ROWS="${SUMMARY_ROWS}| \`$CHART\` | **failed** | 0 | $CHART_SKIP |"$'\n' echo "::error title=httproute $CHART::$CHART has an overlay at ci/httproute/$CHART.yaml but rendered no HTTPRoute, so nothing was checked. That overlay exists only to switch the chart into Gateway mode - either it no longer does (a renamed value, a changed gate) or the chart stopped rendering a route. A green tick here would be a tick for a check that ran over nothing." echo "$CHART: FAILED (overlay present, no HTTPRoute rendered)" else NOT_EXERCISED="$NOT_EXERCISED $CHART" [ "$CHART_SKIP" -gt 0 ] && WITH_GAPS="$WITH_GAPS $CHART" echo "$CHART: no HTTPRoute rendered - nothing to check." fi done # --- Report ------------------------------------------------------------------ count() { set -- $1; echo "$#"; } listing() { set -- $1; [ "$#" -eq 0 ] && { echo "–"; return; }; printf '`%s`' "$1"; shift; for C in "$@"; do printf ', `%s`' "$C"; done; echo; } SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}" { echo "### HTTPRoute render test (Gitea #114)" echo echo "Every \`HTTPRoute\` a chart renders, held against the \`Service\` objects of the" echo "**same** render. An empty \`parentRefs\` attaches the route to no Gateway, and a" echo "\`backendRefs\` entry naming a Service or a port that does not exist resolves to" echo "nothing — both are schema-valid, so neither the real CRDs in the install-test nor" echo "any render error catches them, and the only report would be a route status that no" echo "controller writes in CI." echo echo "| Result | Count | Charts |" echo "|---|---|---|" echo "| HTTPRoutes checked | $(count "$CHECKED") | $(listing "$CHECKED") |" echo "| No HTTPRoute rendered | $(count "$NOT_EXERCISED") | $(listing "$NOT_EXERCISED") |" echo "| Failed | $(count "$FAILED") | $(listing "$FAILED") |" echo echo "$TOTAL_ASSERTIONS assertion(s) held in total." echo echo "**Not provable here, by construction:** that traffic flows. No Gateway controller" echo "runs in CI, so no route ever reaches \`Accepted: True\`." if [ -n "$WITH_GAPS" ]; then echo echo "Charts with references this test does **not** resolve (cross-namespace or non-Service backends, filter-only rules — reported, never counted as passing): $(listing "$WITH_GAPS")." fi if [ -n "$SUMMARY_ROWS" ]; then echo echo "| Chart | Result | Checked | Not checked |" echo "| --- | --- | --- | --- |" printf '%s' "$SUMMARY_ROWS" fi } >>"$SUMMARY" echo "====================== httproute render test ======================" echo "checked: $(count "$CHECKED")${CHECKED:+ -$CHECKED}" echo "not exercised: $(count "$NOT_EXERCISED")${NOT_EXERCISED:+ -$NOT_EXERCISED}" echo "failed: $(count "$FAILED")${FAILED:+ -$FAILED}" echo "assertions held: $TOTAL_ASSERTIONS" echo "==================================================================" if [ -n "$FAILED" ]; then echo "ERROR: HTTPRoute findings in:$FAILED" >&2 exit 1 fi # The second anti-vacuum guard. Reaching zero means every chart fell into "no # HTTPRoute rendered", which is indistinguishable from a broken harness - and # it is the state this test would silently decay into if the overlays were ever # dropped, or if the convention were reverted in the charts without anyone # removing this job. if [ "$TOTAL_ASSERTIONS" -eq 0 ]; then echo "ERROR: not a single HTTPRoute assertion was checked in this run. Either no chart renders an HTTPRoute any more, or the overlays under ci/httproute/ no longer switch one into Gateway mode. A pass over zero assertions is not a pass." >&2 exit 1 fi