helm-charts/ci/run-yaml-scalar-render-test.sh

#!/usr/bin/env bash
# YAML-scalar render test: an operator value that must arrive in the manifest as
# a STRING is held against the manifest it actually produced.
#
# Usage: ci/run-yaml-scalar-render-test.sh [chart ...]   (no argument = all charts)
#
# WHAT IT GUARDS AGAINST (Gitea #11, and #112 turned out to be the same genus)
#
# A value is rendered into a YAML scalar without `| quote`. The operator quotes
# it correctly in their values file, Helm passes the string's CONTENT through,
# and the quotes are gone again in the manifest. Two outcomes, and the quiet one
# is the worse one:
#
#   1. LOUD, but misdirected - the content is not a legal plain scalar and the
#      render aborts. Measured on paperless-ngx' consume CronJob, whose schedule
#      was unquoted: a schedule in the commonest cron notation of all (a star, a
#      slash, an interval) made `helm template` exit 1 with
#
#        error converting YAML to JSON: yaml: line 6: did not find expected
#        alphabetic or numeric character
#
#      YAML read the leading "*" as an alias indicator. The message names the
#      TEMPLATE and the operator's own value is nowhere in it, so the search
#      starts in the wrong place. "@daily" fails the same way.
#   2. SILENT until the API server - the content IS a legal plain scalar, just
#      not a string. "true" becomes a bool, "0755" becomes an integer, and the
#      chart renders perfectly. Kubernetes then rejects the object with
#      "cannot unmarshal number into Go struct field EnvVar.value of type
#      string" - at apply time, which for this repository means in the
#      install-test or in a cluster, never in the render.
#
# Neither is caught by "does helm template exit 0" alone: the first one is, but
# only if something actually renders the object, and the second one is not
# caught at all.
#
# WHAT IS ASSERTED, and all three are properties Kubernetes itself requires -
# there is no chart for which the opposite is a legitimate configuration, which
# is why this needs no per-chart expectations:
#
#   1. CronJob .spec.schedule is a non-empty string.
#   2. Every env[].value of every container and initContainer is a string.
#   3. Every metadata.labels and metadata.annotations value is a string
#      (the API types both as map[string]string).
#
# WHY OVER ALL CHARTS, not the changed ones: the same reason as
# ci/run-quantity-render-test.sh and ci/run-selector-match-test.sh. The env
# block is copied into every chart rather than shared, so the unquoted form
# travels by copy into whatever chart is started from an older one - and that
# chart's own pull request is not where anyone would look for it. Running over
# everything also means there is no list to maintain: adding a chart enrols it.
#
# THE OVERLAYS, AND WHY THIS TEST IS USELESS WITHOUT THEM
#
# The defect above survived in a shipped chart for months for one reason: the
# object that carried it is rendered by NO fixture. paperless-ngx' CronJob sits
# behind smbConnection.enabled, which is false in the chart defaults and in
# every committed fixture, so neither lint, nor the render tests, nor the
# install-test ever produced it. A test that renders only what the fixtures
# already render would have been just as blind.
#
# So ci/yaml-scalar/<chart>.yaml is an overlay applied LAST whose job is to
# switch such objects on AND to feed them the hostile values - a schedule
# beginning with a star, an env value that is a bare "true". Two rules follow,
# the same ones ci/run-selector-match-test.sh works under:
#
#   1. A chart that HAS an overlay and still renders none of the three things
#      above FAILS. The overlay exists for exactly that, so an overlay that
#      stopped reaching its object is a finding and not a shrug.
#   2. If nothing at all was asserted across the whole run, the run fails. A
#      pass over zero assertions is not a pass.
#
# WHAT IS DELIBERATELY NOT CHECKED, so the gap is not mistaken for coverage:
#
#   - Values rendered into a scalar that is hand-quoted in the template
#     ("{{ .Values.x }}"). Those are strings in the output, so this test is
#     blind to them by construction; what they can still do is break the YAML
#     when the value itself contains a quote or a backslash, and only `| quote`
#     fixes that. Reported here rather than silently implied.
#   - Everything that is not a label, an annotation, an env value or a schedule.
#     A bare `storage:` or `className:` is just as unquoted, but its value space
#     is DNS labels and quantities, where no legal value is YAML-ambiguous. The
#     three above are where operator free text actually lands.
#   - Whether a cron schedule is a VALID schedule. That is the cron
#     controller's judgement, and narrowing the schema to a regex was considered
#     and rejected: the defect was never a wrong schedule, it was a correct one
#     rendered bare.
#
# How the YAML is parsed: with helm, via ci/yaml2json, exactly as in
# ci/run-selector-match-test.sh - the runners have neither pyyaml nor yq, and a
# YAML reader written in awk here would fail by quietly reading nothing and
# staying green. That script's header carries the full argument.
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CI_DIR="$ROOT/ci"
YAML2JSON="$CI_DIR/yaml2json"
OVERLAY_DIR="$CI_DIR/yaml-scalar"
RELEASE="scalar"

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

# --- Which charts -----------------------------------------------------------
CHARTS=()
if [ "$#" -gt 0 ]; then
  CHARTS=("$@")
else
  for DIR in "$ROOT"/*/; do
    [ -f "$DIR/Chart.yaml" ] || continue
    CHARTS+=("$(basename "$DIR")")
  done
fi
if [ "${#CHARTS[@]}" -eq 0 ]; then
  echo "ERROR: no chart directories found under $ROOT" >&2
  exit 1
fi

FAILED=""
CHECKED=""
NOT_EXERCISED=""
TOTAL_ASSERTIONS=0
SUMMARY_ROWS=""

for CHART in "${CHARTS[@]}"; do
  if [ ! -f "$ROOT/$CHART/Chart.yaml" ]; then
    echo "ERROR: '$CHART' is not a chart directory" >&2
    exit 1
  fi

  # A chart with a declared dependency cannot be rendered at all while charts/
  # is empty, and this job runs over EVERY chart on EVERY pull request.
  "$ROOT/ci/helm-dependencies.sh" "$ROOT/$CHART"

  VALUES_ARGS=()
  VALUES_LABEL="chart defaults"
  if [ -f "$CI_DIR/$CHART/test-values.yaml" ]; then
    VALUES_ARGS=(-f "$CI_DIR/$CHART/test-values.yaml")
    VALUES_LABEL="ci/$CHART/test-values.yaml"
  elif [ -f "$CI_DIR/trivy/values/$CHART.yaml" ]; then
    VALUES_ARGS=(-f "$CI_DIR/trivy/values/$CHART.yaml")
    VALUES_LABEL="ci/trivy/values/$CHART.yaml"
  fi
  HAS_OVERLAY=""
  if [ -f "$OVERLAY_DIR/$CHART.yaml" ]; then
    # Last, so it wins over the install-test values.
    VALUES_ARGS+=(-f "$OVERLAY_DIR/$CHART.yaml")
    VALUES_LABEL="$VALUES_LABEL + ci/yaml-scalar/$CHART.yaml"
    HAS_OVERLAY=1
  fi

  echo "::group::$CHART ($VALUES_LABEL)"

  RENDER="$WORK/$CHART.manifest.yaml"
  if ! helm template "$RELEASE" "$ROOT/$CHART" \
      ${VALUES_ARGS[@]+"${VALUES_ARGS[@]}"} >"$RENDER" 2>"$WORK/err"; then
    sed 's/^/  /' "$WORK/err"
    echo "::endgroup::"
    echo "::error title=yaml-scalar $CHART::$CHART does not render with these values. If the error is a YAML parse error naming a template line, that IS the finding this test exists for: a value reached a scalar unquoted. Reproduce: helm template $RELEASE $CHART $(printf '%s ' "${VALUES_ARGS[@]+${VALUES_ARGS[@]}}")"
    FAILED="$FAILED $CHART"
    continue
  fi

  # The document split, the only textual step. A wrong split cannot produce a
  # false pass: the result stops being parsable and helm exits 1.
  DOCS="$WORK/$CHART.docs.yaml"
  awk '
    BEGIN { print "docs:"; started = 0 }
    /^---[ \t]*$/ { print "  -"; started = 1; next }
    {
      if (!started) { print "  -"; started = 1 }
      if (length($0) == 0) print ""; else print "    " $0
    }
  ' "$RENDER" >"$DOCS"

  JSON="$WORK/$CHART.json"
  if ! helm template y2j "$YAML2JSON" -f "$DOCS" 2>"$WORK/err" \
      | awk '/^\{/ { print; exit }' >"$JSON"; then
    sed 's/^/  /' "$WORK/err"
    echo "::endgroup::"
    echo "::error title=yaml-scalar $CHART::the rendered manifests of $CHART could not be re-read as YAML by ci/yaml2json. That is a parse failure, not a scalar finding."
    FAILED="$FAILED $CHART"
    continue
  fi
  if [ ! -s "$JSON" ]; then
    echo "::endgroup::"
    echo "::error title=yaml-scalar $CHART::ci/yaml2json produced no JSON for $CHART - the manifests were not re-read, so nothing was checked."
    FAILED="$FAILED $CHART"
    continue
  fi

  # --- The comparison --------------------------------------------------------
  # Standard library only, and it never decides the exit code itself: it writes
  # "VERDICT|detail" lines and the loop below decides, so a verdict can never
  # be lost to a swallowed status.
  set +e
  python3 - "$CHART" "$JSON" >"$WORK/$CHART.out" 2>"$WORK/$CHART.pyerr" <<'PY'
import json, sys

chart, path = sys.argv[1], sys.argv[2]
docs = json.load(open(path)).get("docs") or []
lines = []


def name_of(doc):
    meta = doc.get("metadata")
    if isinstance(meta, dict) and meta.get("name"):
        return str(meta["name"])
    return "<unnamed %s>" % doc.get("kind", "object")


def typename(value):
    return {bool: "bool", int: "int", float: "float", type(None): "null",
            dict: "mapping", list: "list"}.get(type(value), type(value).__name__)


def pod_templates(node, path="$"):
    """Any value of a key named "template" carrying a spec with containers.
    Found by walking, so a CronJob's spec.jobTemplate.spec.template needs no
    special case and neither does whatever workload kind a chart adds next."""
    out = []
    if isinstance(node, dict):
        for key, value in node.items():
            here = "%s.%s" % (path, key)
            if (key == "template" and isinstance(value, dict)
                    and isinstance(value.get("spec"), dict)
                    and "containers" in value["spec"]):
                out.append((here, value))
            out.extend(pod_templates(value, here))
    elif isinstance(node, list):
        for i, value in enumerate(node):
            out.extend(pod_templates(value, "%s[%d]" % (path, i)))
    return out


def metadatas(node, path="$"):
    """Every metadata mapping anywhere in the render - top level and inside pod
    templates both, because labels and annotations exist at both levels."""
    out = []
    if isinstance(node, dict):
        for key, value in node.items():
            here = "%s.%s" % (path, key)
            if key == "metadata" and isinstance(value, dict):
                out.append((here, value))
            out.extend(metadatas(value, here))
    elif isinstance(node, list):
        for i, value in enumerate(node):
            out.extend(metadatas(value, "%s[%d]" % (path, i)))
    return out


# --- 1. CronJob schedules ---------------------------------------------------
for doc in docs:
    if not isinstance(doc, dict) or doc.get("kind") != "CronJob":
        continue
    where = "CronJob %s .spec.schedule" % name_of(doc)
    spec = doc.get("spec") if isinstance(doc.get("spec"), dict) else {}
    schedule = spec.get("schedule", "<absent>")
    if isinstance(schedule, str) and schedule.strip():
        lines.append("OK|%s: %s is the string %r" % (chart, where, schedule))
    elif schedule == "<absent>":
        lines.append("FAIL|%s: %s is missing entirely - a CronJob without a "
                     "schedule is rejected by the API server" % (chart, where))
    else:
        lines.append("FAIL|%s: %s rendered as %s (%r), not a string. A schedule "
                     "is operator free text and has to leave the template "
                     "through | quote; without it YAML re-reads the content and "
                     "a schedule beginning with a digit survives by luck while "
                     "one beginning with * or @ does not"
                     % (chart, where, typename(schedule), schedule))

# --- 2. env values ----------------------------------------------------------
for where, template in pod_templates({"docs": docs}):
    spec = template.get("spec") if isinstance(template.get("spec"), dict) else {}
    for kind in ("initContainers", "containers"):
        for container in spec.get(kind) or []:
            if not isinstance(container, dict):
                continue
            cname = container.get("name", "<unnamed>")
            for entry in container.get("env") or []:
                if not isinstance(entry, dict) or "value" not in entry:
                    continue  # valueFrom entries carry no literal to check
                value = entry["value"]
                tag = "env %s of container %s at %s" % (
                    entry.get("name", "<unnamed>"), cname, where)
                if isinstance(value, str):
                    lines.append("OK|%s: %s is a string" % (chart, tag))
                else:
                    lines.append("FAIL|%s: %s rendered as %s (%r), not a string. "
                                 "Kubernetes types env[].value as a string and "
                                 "the API server rejects the whole manifest with "
                                 "\"cannot unmarshal %s into Go struct field "
                                 "EnvVar.value of type string\" - at apply time, "
                                 "not in the render. The fix is | quote at the "
                                 "render site; template-chart carries the form"
                                 % (chart, tag, typename(value), value,
                                    "number" if not isinstance(value, bool) else "bool"))

# --- 3. labels and annotations ----------------------------------------------
for where, meta in metadatas({"docs": docs}):
    for field in ("labels", "annotations"):
        block = meta.get(field)
        if not isinstance(block, dict):
            continue
        for key, value in block.items():
            tag = "%s[%r] at %s" % (field, key, where)
            if isinstance(value, str):
                lines.append("OK|%s: %s is a string" % (chart, tag))
            else:
                lines.append("FAIL|%s: %s rendered as %s (%r), not a string. "
                             "Kubernetes types %s as map[string]string, so the "
                             "API server rejects the object - the render stays "
                             "green. The fix is | quote at the render site"
                             % (chart, tag, typename(value), value, field))

for line in lines:
    print(line)
PY
  RC=$?
  set -e
  if [ "$RC" -gt 1 ]; then
    sed 's/^/  /' "$WORK/$CHART.pyerr"
    echo "::endgroup::"
    echo "::error title=yaml-scalar $CHART::the scalar comparison itself failed for $CHART (python exit $RC). Treated as a failure, never as a pass."
    FAILED="$FAILED $CHART"
    continue
  fi

  CHART_FAIL=""
  CHART_OK=0
  while IFS='|' read -r VERDICT DETAIL; do
    case "$VERDICT" in
      OK)
        CHART_OK=$((CHART_OK + 1))
        echo "  ok     $DETAIL"
        ;;
      FAIL)
        CHART_FAIL=1
        echo "  FAIL   $DETAIL"
        echo "::error title=yaml-scalar $CHART::$DETAIL"
        ;;
      *)
        [ -n "$VERDICT" ] && echo "  $VERDICT $DETAIL"
        ;;
    esac
  done <"$WORK/$CHART.out"
  echo "::endgroup::"

  TOTAL_ASSERTIONS=$((TOTAL_ASSERTIONS + CHART_OK))

  if [ -n "$CHART_FAIL" ]; then
    FAILED="$FAILED $CHART"
    SUMMARY_ROWS="${SUMMARY_ROWS}| \`$CHART\` | **failed** | $CHART_OK |"$'\n'
    echo "$CHART: FAILED"
  elif [ "$CHART_OK" -gt 0 ]; then
    CHECKED="$CHECKED $CHART"
    SUMMARY_ROWS="${SUMMARY_ROWS}| \`$CHART\` | ok | $CHART_OK |"$'\n'
    echo "$CHART: ok ($CHART_OK scalar(s) checked)"
  elif [ -n "$HAS_OVERLAY" ]; then
    FAILED="$FAILED $CHART"
    SUMMARY_ROWS="${SUMMARY_ROWS}| \`$CHART\` | **failed** | 0 |"$'\n'
    echo "::error title=yaml-scalar $CHART::$CHART has an overlay at ci/yaml-scalar/$CHART.yaml but its render carried no schedule, no env value and no label or annotation at all, so nothing was checked. That overlay exists only to make such a scalar render; a green tick here would be a tick for a check that ran over nothing."
    echo "$CHART: FAILED (overlay present, nothing asserted)"
  else
    NOT_EXERCISED="$NOT_EXERCISED $CHART"
    echo "$CHART: nothing of the three kinds rendered - nothing to check."
  fi
done

# --- Report ------------------------------------------------------------------
count() { set -- $1; echo "$#"; }
listing() { set -- $1; [ "$#" -eq 0 ] && { echo "–"; return; }; printf '`%s`' "$1"; shift; for C in "$@"; do printf ', `%s`' "$C"; done; echo; }
SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
{
  echo "### YAML-scalar render test (Gitea #11)"
  echo
  echo "Every \`CronJob.spec.schedule\`, every \`env[].value\` and every label and"
  echo "annotation value in a chart's render, held against the type Kubernetes requires"
  echo "for it. A value rendered without \`| quote\` either aborts the render with an"
  echo "error pointing at the template instead of at the value, or arrives as a bool or"
  echo "an integer and is rejected by the API server at apply time - never in the render."
  echo
  echo "| Result | Count | Charts |"
  echo "|---|---|---|"
  echo "| Scalars checked | $(count "$CHECKED") | $(listing "$CHECKED") |"
  echo "| Nothing of the kind rendered | $(count "$NOT_EXERCISED") | $(listing "$NOT_EXERCISED") |"
  echo "| Failed | $(count "$FAILED") | $(listing "$FAILED") |"
  echo
  echo "$TOTAL_ASSERTIONS scalar(s) verified in total."
  if [ -n "$SUMMARY_ROWS" ]; then
    echo
    echo "| Chart | Result | Checked |"
    echo "| --- | --- | --- |"
    printf '%s' "$SUMMARY_ROWS"
  fi
} >>"$SUMMARY"

echo "===================== yaml scalar render test ======================"
echo "checked:       $(count "$CHECKED")${CHECKED:+ -$CHECKED}"
echo "not exercised: $(count "$NOT_EXERCISED")${NOT_EXERCISED:+ -$NOT_EXERCISED}"
echo "failed:        $(count "$FAILED")${FAILED:+ -$FAILED}"
echo "scalars verified: $TOTAL_ASSERTIONS"
echo "==================================================================="

if [ -n "$FAILED" ]; then
  echo "ERROR: a value that has to arrive as a string did not, in:$FAILED" >&2
  exit 1
fi

if [ "$TOTAL_ASSERTIONS" -eq 0 ]; then
  echo "ERROR: not a single scalar was checked in this run. Either no chart renders a schedule, an env value, a label or an annotation any more, or the overlays under ci/yaml-scalar/ no longer reach their objects. A pass over zero assertions is not a pass." >&2
  exit 1
fi