From d8b392bf64cb6c8715bf17ce33ab7f59ede2c93c Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 10:26:13 +0000 Subject: [PATCH 1/8] feat(keycloak): extraVolumes/extraVolumeMounts for custom themes and providers (1.1.0+up26.8.0) --- ci/keycloak/fixtures.yaml | 22 ++++ ci/keycloak/test-values.yaml | 45 +++++++++ keycloak/Chart.yaml | 2 +- keycloak/templates/keycloak.deployment.yaml | 32 ++++++ keycloak/values.schema.json | 10 +- keycloak/values.yaml | 105 ++++++++++++++++++++ 6 files changed, 214 insertions(+), 2 deletions(-) diff --git a/ci/keycloak/fixtures.yaml b/ci/keycloak/fixtures.yaml index 976bd2e..a5c6a3f 100644 --- a/ci/keycloak/fixtures.yaml +++ b/ci/keycloak/fixtures.yaml @@ -85,6 +85,28 @@ spec: emptyDir: {} --- apiVersion: v1 +kind: ConfigMap +metadata: + # A MINIMAL BUT REAL LOGIN THEME, so that the install-test actually walks the + # path keycloak.extraVolumes exists for (Gitea #88) instead of only rendering + # it. ci/keycloak/test-values.yaml mounts it on the SUBPATH + # /opt/keycloak/themes/ci-theme - never on /opt/keycloak/themes itself, which + # would hide the image's base, keycloak and keycloak.v2 themes. + name: ci-keycloak-theme +data: + # A theme's layout is //theme.properties, and a ConfigMap key + # cannot contain a slash - so the "login/" directory is produced by + # items[].path in the volume, which can. That indirection is exactly the part + # of the theme route a render cannot verify and a running pod can. + theme.properties: | + parent=keycloak.v2 + # The reason the issue was opened: Keycloak has no logo field on an identity + # provider, so the icon on its login button comes only from a login theme. + # Nothing in CI drives a browser, so this line is carried rather than + # checked - what the run proves is that the theme reaches the container. + kcLogoIdP-ci-provider = fa fa-lock +--- +apiVersion: v1 kind: Service metadata: name: postgres diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index 7fe8170..1de14ef 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -29,6 +29,9 @@ # that crashed, deadlocked on the schema migration lock or fought the # first over the database would fail the run. Measured in Gitea PR #87: # 71.8s from `helm install` to both Ready, 0 restarts. +# 5. A MOUNTED LOGIN THEME SURVIVES ALL OF THAT (Gitea #88) - the full +# argument sits at keycloak.extraVolumes further down, including what it +# still does not prove. # Plus the mechanical half: the Secret key names match what the Deployment # references, /health/started, /health/live and /health/ready all answer on # the management port (so health-enabled really was baked in by the init @@ -124,6 +127,48 @@ keycloak: memory: 512Mi ephemeral-storage: 256Mi + # THE EXTRA VOLUME PAIR, WALKED RATHER THAN RENDERED (Gitea #88). The theme + # ConfigMap comes from fixtures.yaml. Three things this puts inside the + # assertion that no render reaches: + # + # 1. A MOUNT ON A SUBPATH OF A DIRECTORY THAT EXISTS IN THE IMAGE IS + # ACCEPTED ALONGSIDE readOnlyRootFilesystem: true. + # /opt/keycloak/themes/ci-theme does not exist in the image, so the + # runtime has to create the mount point inside a root filesystem that is + # about to be read-only. If that did not work, every container of this + # pod would fail to start. + # 2. `kc.sh build` STILL SUCCEEDS WITH THE EXTRA MOUNT PRESENT. The mounts + # are rendered on the build init container too (that is what makes SPI + # providers work at all), so this fixture is also the test that the extra + # mount does not disturb the augmentation - a failure would leave the pod + # in Init with the reason only in a log. + # 3. `start --optimized` COMES UP WITH A RUNTIME THEME IN PLACE. The plan of + # #88 called that "a reasoned expectation, not verified"; a Ready pod with + # this mounted is the verification, and it is the one that matters for the + # unusual two-init-container construction of this chart. + # + # WHAT IT STILL DOES NOT PROVE, and it would be easy to overclaim here: that + # the LOGIN PAGE renders and that base/keycloak/keycloak.v2 are still listed. + # The health endpoints answer regardless of the themes directory, and nothing + # in this harness makes an HTTP request to / or runs a command in the pod. That + # the subpath hides nothing is a property of the mount path rather than a + # measurement - and the kcLogoIdP line needs a browser plus a realm setting, + # neither of which exists in CI. + extraVolumes: + - name: login-theme + configMap: + name: ci-keycloak-theme + items: + - key: theme.properties + path: login/theme.properties + extraVolumeMounts: + - name: login-theme + # THE SUBPATH, AND THE WHOLE POINT OF THE EXAMPLE: a volume on + # /opt/keycloak/themes would hide the image's own three themes and break + # the login page while the pod stayed Ready. + mountPath: /opt/keycloak/themes/ci-theme + readOnly: true + ingress: # Left ON although no ingress controller answers in CI: the point is that # both objects are accepted and that the Service port they name BY NAME diff --git a/keycloak/Chart.yaml b/keycloak/Chart.yaml index 0752bf9..c54212f 100644 --- a/keycloak/Chart.yaml +++ b/keycloak/Chart.yaml @@ -43,7 +43,7 @@ type: application # procedure lives in NOTES.txt and on the wiki page instead of in a # packageRules entry. Read the "MINOR UPGRADES" section of NOTES.txt before # merging an app-version PR that changes the minor. -version: 1.0.0+up26.8.0 +version: 1.1.0+up26.8.0 appVersion: "26.8.0" maintainers: diff --git a/keycloak/templates/keycloak.deployment.yaml b/keycloak/templates/keycloak.deployment.yaml index 6bdb618..2a17fe2 100644 --- a/keycloak/templates/keycloak.deployment.yaml +++ b/keycloak/templates/keycloak.deployment.yaml @@ -146,6 +146,23 @@ spec: mountPath: {{ include "keycloak.dataDir" . }} - name: tmp mountPath: /tmp + {{- with .Values.keycloak.extraVolumeMounts }} + # THE EXTRA MOUNTS, HERE AS WELL AS ON THE SERVING CONTAINER, AND + # THIS OCCURRENCE IS THE LOAD-BEARING ONE (Gitea #88). Do not + # "clean it up" to the serving container only: an SPI provider JAR + # is a BUILD-time concern. Upstream, on adding one: "After that if + # you are using `--optimized`, you must run the `build` command in + # order to update the server's provider registry with the + # implementations from the JAR file." That build is THIS container, + # so a JAR mounted only below is absent while the registry is + # written - it never gets registered and nothing reports it. + # + # One list for both places on purpose: two lists would be two + # chances to mount an extension in the wrong one, and the wrong one + # fails silently. The price is that every extra volume is present at + # build time too, which values.yaml names at the value. + {{- toYaml . | nindent 12 }} + {{- end }} containers: - name: {{ .Release.Name }}-{{ .Chart.Name }}-deployment # Fully qualified: an unqualified reference would resolve to Docker @@ -358,6 +375,13 @@ spec: mountPath: {{ include "keycloak.dataDir" . }} - name: tmp mountPath: /tmp + {{- with .Values.keycloak.extraVolumeMounts }} + # The extra mounts, second of the two places (see the build init + # container above). This is where a login THEME is read - themes are + # a runtime concern, so one mounted here takes effect even though + # this container serves a pre-built installation. + {{- toYaml . | nindent 12 }} + {{- end }} # DELIBERATELY NOT MOUNTED: /opt/keycloak/conf. It is not empty in # the image - it carries keycloak.conf and, more importantly, # cache-ispn.xml, which is the Infinispan configuration cache=ispn @@ -371,3 +395,11 @@ spec: emptyDir: {} - name: tmp emptyDir: {} + {{- with .Values.keycloak.extraVolumes }} + # Verbatim, because a volume source can be any of the Kubernetes forms + # and the chart interprets none of them. The one path rule that matters + # is about the MOUNT and not the volume: never /opt/keycloak/themes + # itself, which would hide the image's base, keycloak and keycloak.v2 + # themes - see values.yaml at keycloak.extraVolumes. + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/keycloak/values.schema.json b/keycloak/values.schema.json index 7ab8e86..b5499b2 100644 --- a/keycloak/values.schema.json +++ b/keycloak/values.schema.json @@ -143,7 +143,15 @@ "minAvailable": { "type": "integer", "minimum": 1 } } }, - "env": { "$ref": "#/definitions/env" } + "env": { "$ref": "#/definitions/env" }, + "extraVolumes": { + "description": "Additional pod volumes, rendered verbatim into the pod spec. The ITEMS stay loosely typed on purpose - a volume source can be any of the Kubernetes forms (configMap, secret, persistentVolumeClaim, emptyDir, projected, csi, ...), and a stricter schema here would reject valid configurations rather than catch mistakes; the API server validates the shape anyway. Declaring the key itself is not optional, because the levels above carry additionalProperties: false. Places files, configures nothing: an extra root CA additionally needs KC_TRUSTSTORE_PATHS (a runtime option, so keycloak.env) and a theme additionally needs to be selected per realm in Keycloak's database.", + "type": ["array", "null"] + }, + "extraVolumeMounts": { + "description": "Additional volume mounts, rendered verbatim and TWICE - on the build init container as well as on the serving container (Gitea #88). That is deliberate: a login theme is read at runtime, but an SPI provider JAR must be present while `kc.sh build` writes the provider registry, or it is never registered and nothing reports it. Items loosely typed for the same reason as extraVolumes. Never mount on /opt/keycloak/themes itself - it would hide the image's base, keycloak and keycloak.v2 themes; use a subpath /opt/keycloak/themes/.", + "type": ["array", "null"] + } } }, "ingress": { diff --git a/keycloak/values.yaml b/keycloak/values.yaml index afdf5ae..eeacc88 100644 --- a/keycloak/values.yaml +++ b/keycloak/values.yaml @@ -509,6 +509,111 @@ keycloak: # KC_SPI_* provider settings. env: [] + # ADDITIONAL VOLUMES AND THEIR MOUNTS - the way a file gets into this pod at + # all (Gitea #88). Deliberately GENERIC rather than a theme-specific key: + # the same pair serves a custom login theme, an SPI provider JAR and an extra + # root CA, and the chart does not have to decide how a theme is delivered + # (ConfigMap, Secret, PVC, an init container of your own filling an + # emptyDir). Both lists are passed through verbatim, so an entry may use any + # Kubernetes volume source. + # + # THE MOUNTS ARE RENDERED TWICE - ON THE build INIT CONTAINER AND ON THE + # SERVING CONTAINER - AND THAT IS THE POINT, NOT AN OVERSIGHT. One list, two + # places, because the two cases are resolved at different times: + # + # * The SERVING container is where a THEME is read. Themes are a runtime + # concern, so a theme mounted there takes effect in the running server + # even though this chart serves a pre-built installation. + # * The build INIT container is where `kc.sh build` runs (point 2 at the + # top of this file), and a PROVIDER is a build-time concern. Upstream, on + # dropping a provider JAR into providers/: "After that if you are using + # `--optimized`, you must run the `build` command in order to update the + # server's provider registry with the implementations from the JAR file." + # This chart runs `start --optimized`, so a JAR that is mounted only on + # the serving container is absent while the registry is written: it never + # gets registered, the file sits there at runtime, and NOTHING reports + # it - no error, no warning, just an extension that does nothing. + # + # Not on the SEED init container: it has a different layout (it only writes + # the shared volume at /quarkus-build and runs no Keycloak command), so a + # mount there would be pointless. + # + # The side effect of the second place, said out loud rather than discovered: + # an extra volume is present AT BUILD TIME as well. For a theme that is + # without consequence. For a Secret it means an init container of this pod + # sees it too - the same pod either way, but it belongs stated. + # + # WHAT THIS PAIR DOES NOT DO: it places files, it does not configure + # Keycloak. Two cases where that difference is the whole thing: + # + # * AN EXTRA ROOT CA IS NOT TRUSTED BY BEING MOUNTED. A file in the pod is + # in no truststore; Keycloak reads one only from truststore-paths, so the + # mount has to be accompanied by KC_TRUSTSTORE_PATHS pointing at the + # mounted DIRECTORY. That is a RUNTIME option (checked against the 26.8.0 + # option table, where it carries no build icon), so keycloak.env above is + # the right place for it and the build-option guard does not refuse it. + # * A THEME IS NOT ACTIVATED BY BEING MOUNTED, AND THIS CHART CANNOT + # ACTIVATE IT. The selection lives PER REALM in Keycloak's database + # (Realm settings -> Themes -> Login theme). Mounting makes a theme + # selectable, never selected - so do not go looking for a bug when a + # freshly mounted theme changes nothing. + # + # THE THEME CASE, AND THE ONE MISTAKE EVERYBODY MAKES EXACTLY ONCE: + # + # DO NOT MOUNT ON /opt/keycloak/themes. The image's own themes live there + # (base, keycloak, keycloak.v2) and a volume at that path HIDES ALL THREE - + # the login page is then broken rather than merely unstyled. Mount on a + # SUBPATH that does not exist in the image, /opt/keycloak/themes/, + # which hides nothing. Same mechanism and same class of mistake as the + # "DELIBERATELY NOT MOUNTED: /opt/keycloak/conf" note in the Deployment, + # where an emptyDir would hide the cache-ispn.xml that cache=ispn reads. + # + # extraVolumes: + # - name: login-theme + # configMap: + # name: my-login-theme + # # A theme's own layout is //theme.properties, and a + # # ConfigMap key may not contain a slash - so the subdirectory comes + # # from items[].path, which may. + # items: + # - key: theme.properties + # path: login/theme.properties + # extraVolumeMounts: + # - name: login-theme + # # NOT /opt/keycloak/themes - see above. + # mountPath: /opt/keycloak/themes/my-theme + # readOnly: true + # + # With that in place a theme.properties carrying + # + # parent=keycloak.v2 + # kcLogoIdP- = fa fa-lock + # + # is what puts an icon on an identity provider's login button - Keycloak has + # no logo field on a provider, so a login theme is the only way to set one. + # + # FOUR PROPERTIES OF THE THEME ROUTE THAT COST TIME IF THEY ARE NOT KNOWN: + # + # 1. A ConfigMap carries binary content ONLY in binaryData (base64). An SVG + # is text and fits in data; a PNG does not. And the 1 MiB limit per + # ConfigMap applies - ample for a theme with SVG icons, not enough for + # bundled web fonts, which belong in a PVC or an image of your own. + # 2. THEMES ARE CACHED, and in production that is wanted. It means a theme + # change takes effect only after a pod restart. Two things add up here: + # this chart hashes no user-supplied ConfigMap either (it renders none of + # its own - point 1 at the top of this file), so editing the theme + # ConfigMap restarts nothing by itself. Use + # `kubectl rollout restart deploy/-keycloak-deployment`. + # Do not configure the cache away to avoid this. + # 3. The realm selection is not ours to make - see above. + # 4. A CUSTOM LOGIN THEME CAN MAKE THE IDENTITY-PROVIDER BUTTONS DISAPPEAR + # ENTIRELY when its template does not include the social fragment + # correctly (reported repeatedly upstream, keycloak/keycloak#42711). + # After switching a realm's theme, LOOK AT THE LOGIN PAGE - a healthy + # pod says nothing about whether the buttons are still there. + extraVolumes: [] + extraVolumeMounts: [] + ingress: # The PUBLIC ingress - the login endpoint, the token endpoints, the account # console. On by default, because an identity provider nothing can reach is -- 2.54.0 From 1cf59f5448a4b27c5ede9cf9aba7b4ee69612963 Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 10:46:57 +0000 Subject: [PATCH 2/8] docs(ci): record the keycloak install-test run that was slower and restarted once --- ci/keycloak/test-values.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index 1de14ef..55d0dc7 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -28,7 +28,20 @@ # rollout is complete only when BOTH pods are Ready, so a second replica # that crashed, deadlocked on the schema migration lock or fought the # first over the database would fail the run. Measured in Gitea PR #87: -# 71.8s from `helm install` to both Ready, 0 restarts. +# 71.8s from `helm install` to both Ready, 0 restarts - three runs, all +# three 72s and 0 restarts. +# THE FOURTH RUN WAS NOT THAT, AND THE NUMBER IS RECORDED RATHER THAN +# SMOOTHED (Gitea PR #92, run 585): 101s, and ONE restart on one of the two +# pods. The same run was measurably slower everywhere - the UNCHANGED +# postgres fixture took 51.5s to roll out instead of 27.4s - so the runner +# was about half as fast, and this chart's first install is where that +# shows: two replicas serialize on the Liquibase migration lock, which is +# the whole reason settings.env budgets 900s. WHAT THE RESTART WAS HAS NOT +# BEEN ESTABLISHED, because the harness dumps container logs only when a +# run FAILS, and this one passed. So treat "0 restarts" as an observation +# from three fast runs rather than as an invariant of the chart, and if a +# run ever needs the reason, the harness has to print it on the green path +# too. # 5. A MOUNTED LOGIN THEME SURVIVES ALL OF THAT (Gitea #88) - the full # argument sits at keycloak.extraVolumes further down, including what it # still does not prove. -- 2.54.0 From 34c600f7a8aa457ab755f9583a7eea02b7d750bc Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 10:54:48 +0000 Subject: [PATCH 3/8] docs(ci): the second run settles it - the keycloak restart was the runner, not the pod spec --- ci/keycloak/test-values.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index 55d0dc7..b8583b6 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -38,10 +38,16 @@ # shows: two replicas serialize on the Liquibase migration lock, which is # the whole reason settings.env budgets 900s. WHAT THE RESTART WAS HAS NOT # BEEN ESTABLISHED, because the harness dumps container logs only when a -# run FAILS, and this one passed. So treat "0 restarts" as an observation -# from three fast runs rather than as an invariant of the chart, and if a -# run ever needs the reason, the harness has to print it on the green path -# too. +# run FAILS, and this one passed - which is the lasting finding here: the +# green path cannot explain a restart it reports. +# WHAT DID SETTLE THE QUESTION WAS A SECOND RUN OF THE IDENTICAL MANIFESTS +# (run 590): 82s, 0 restarts, postgres back to 23.4s. Same chart, same +# theme mount, fast runner - so the restart belongs to the environment and +# not to the pod spec. Keep both numbers: "0 restarts" is an observation +# about four runs on a runner of varying speed, not an invariant of the +# chart, and a first install that serializes two replicas on a schema +# migration lock is exactly the shape that gets slower-than-linearly worse +# on a loaded machine. # 5. A MOUNTED LOGIN THEME SURVIVES ALL OF THAT (Gitea #88) - the full # argument sits at keycloak.extraVolumes further down, including what it # still does not prove. -- 2.54.0 From 5e47191252550ee75201f0e6b4aa50c8b29e4d9c Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 11:04:16 +0000 Subject: [PATCH 4/8] docs(ci): three runs, and the load explanation does not hold --- ci/keycloak/test-values.yaml | 38 +++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index b8583b6..4f9a76a 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -30,24 +30,26 @@ # first over the database would fail the run. Measured in Gitea PR #87: # 71.8s from `helm install` to both Ready, 0 restarts - three runs, all # three 72s and 0 restarts. -# THE FOURTH RUN WAS NOT THAT, AND THE NUMBER IS RECORDED RATHER THAN -# SMOOTHED (Gitea PR #92, run 585): 101s, and ONE restart on one of the two -# pods. The same run was measurably slower everywhere - the UNCHANGED -# postgres fixture took 51.5s to roll out instead of 27.4s - so the runner -# was about half as fast, and this chart's first install is where that -# shows: two replicas serialize on the Liquibase migration lock, which is -# the whole reason settings.env budgets 900s. WHAT THE RESTART WAS HAS NOT -# BEEN ESTABLISHED, because the harness dumps container logs only when a -# run FAILS, and this one passed - which is the lasting finding here: the -# green path cannot explain a restart it reports. -# WHAT DID SETTLE THE QUESTION WAS A SECOND RUN OF THE IDENTICAL MANIFESTS -# (run 590): 82s, 0 restarts, postgres back to 23.4s. Same chart, same -# theme mount, fast runner - so the restart belongs to the environment and -# not to the pod spec. Keep both numbers: "0 restarts" is an observation -# about four runs on a runner of varying speed, not an invariant of the -# chart, and a first install that serializes two replicas on a schema -# migration lock is exactly the shape that gets slower-than-linearly worse -# on a loaded machine. +# THE NEXT THREE RUNS WERE NOT THAT, AND THE NUMBERS ARE RECORDED RATHER +# THAN SMOOTHED (Gitea PR #92, same manifests in all three - the two later +# commits change only comments): +# run 585: 101s, ONE restart on one of the two pods +# run 590: 82s, 0 restarts +# run 593: 191s, 0 restarts +# So the restart is NOT caused by the pod spec: the fastest and the slowest +# run both had none, and the run that had one sat between them. The first +# explanation that offered itself does not survive the third run either - +# run 585 was slower everywhere (the UNCHANGED postgres fixture took 51.5s +# instead of 27.4s), but run 593 was slower still and restarted nothing, so +# "a loaded runner trips the first install" is not what was measured. +# WHAT THE RESTART WAS HAS NOT BEEN ESTABLISHED, and that is the lasting +# finding: the harness dumps container logs only when a run FAILS +# (diagnose() hangs off the EXIT trap behind "RC -ne 0"), and all three runs +# passed - A GREEN RUN CANNOT EXPLAIN A RESTART IT REPORTS. Whoever next +# needs that answer has to make the final-state step print the restart +# reasons when any counter is above zero. +# Read "0 restarts" accordingly: an observation about six runs on a runner +# of varying speed, not an invariant of this chart. # 5. A MOUNTED LOGIN THEME SURVIVES ALL OF THAT (Gitea #88) - the full # argument sits at keycloak.extraVolumes further down, including what it # still does not prove. -- 2.54.0 From d7067abfabfa82be4b8741ffa633fecd77316e6d Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 11:13:46 +0000 Subject: [PATCH 5/8] docs(ci): the whole restart series, intermittent on an unchanged spec --- ci/keycloak/test-values.yaml | 40 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index 4f9a76a..028808a 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -30,26 +30,32 @@ # first over the database would fail the run. Measured in Gitea PR #87: # 71.8s from `helm install` to both Ready, 0 restarts - three runs, all # three 72s and 0 restarts. -# THE NEXT THREE RUNS WERE NOT THAT, AND THE NUMBERS ARE RECORDED RATHER -# THAN SMOOTHED (Gitea PR #92, same manifests in all three - the two later -# commits change only comments): +# THE RUNS OF PR #92 WERE NOT THAT, AND THE NUMBERS ARE RECORDED RATHER +# THAN SMOOTHED. Identical pod spec in all four - the later commits on that +# branch change comments only: # run 585: 101s, ONE restart on one of the two pods # run 590: 82s, 0 restarts # run 593: 191s, 0 restarts -# So the restart is NOT caused by the pod spec: the fastest and the slowest -# run both had none, and the run that had one sat between them. The first -# explanation that offered itself does not survive the third run either - -# run 585 was slower everywhere (the UNCHANGED postgres fixture took 51.5s -# instead of 27.4s), but run 593 was slower still and restarted nothing, so -# "a loaded runner trips the first install" is not what was measured. -# WHAT THE RESTART WAS HAS NOT BEEN ESTABLISHED, and that is the lasting -# finding: the harness dumps container logs only when a run FAILS -# (diagnose() hangs off the EXIT trap behind "RC -ne 0"), and all three runs -# passed - A GREEN RUN CANNOT EXPLAIN A RESTART IT REPORTS. Whoever next -# needs that answer has to make the final-state step print the restart -# reasons when any counter is above zero. -# Read "0 restarts" accordingly: an observation about six runs on a runner -# of varying speed, not an invariant of this chart. +# run 595: 101s, ONE restart on one of the two pods +# Three things that are worth more than the average: the restart is +# INTERMITTENT on an unchanged spec (2 of 4), it is always exactly ONE on +# ONE pod, about 35s before that pod reports Ready - and it does not track +# the runner's speed, which was the first explanation and does not hold: +# the FASTEST (82s) and the SLOWEST (191s) run restarted nothing, and the +# two that restarted were both 101s. +# WHAT THE RESTART IS HAS NOT BEEN ESTABLISHED, and that is the lasting +# finding of the series: the harness dumps container logs only when a run +# FAILS (diagnose() hangs off the EXIT trap behind "RC -ne 0") and all four +# passed - A GREEN RUN CANNOT EXPLAIN A RESTART IT REPORTS. Answering it +# means making the final-state step print the restart reasons whenever a +# counter is above zero; that is a change to ci/run-install-test.sh and it +# was deliberately NOT made inside #88. +# The counting stops here on purpose - every further commit adds a run, and +# a comment that chases the tally is wrong at the next push. What is worth +# carrying is the shape, not the tally: "0 restarts" is an observation, not +# an invariant of this chart, and a first install that serializes two +# replicas on a schema migration lock is a plausible place for one - which +# is a hypothesis, stated as one, because nothing here measured it. # 5. A MOUNTED LOGIN THEME SURVIVES ALL OF THAT (Gitea #88) - the full # argument sits at keycloak.extraVolumes further down, including what it # still does not prove. -- 2.54.0 From 71cc5452775b62ffeafbb9091d21237a6ae6b469 Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 11:22:26 +0000 Subject: [PATCH 6/8] test(ci): A/B control run - keycloak install-test without the theme mount (temporary) --- ci/keycloak/test-values.yaml | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index 028808a..d3d7af1 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -181,20 +181,25 @@ keycloak: # the subpath hides nothing is a property of the mount path rather than a # measurement - and the kcLogoIdP line needs a browser plus a realm setting, # neither of which exists in CI. - extraVolumes: - - name: login-theme - configMap: - name: ci-keycloak-theme - items: - - key: theme.properties - path: login/theme.properties - extraVolumeMounts: - - name: login-theme - # THE SUBPATH, AND THE WHOLE POINT OF THE EXAMPLE: a volume on - # /opt/keycloak/themes would hide the image's own three themes and break - # the login page while the pod stayed Ready. - mountPath: /opt/keycloak/themes/ci-theme - readOnly: true + # TEMPORARILY DISABLED FOR AN A/B MEASUREMENT, RESTORED BEFORE THIS PR IS + # MERGED. Three of five runs with the mount restarted one pod once, while the + # three runs of PR #87 without it restarted none - too close to be dismissed + # and not establishable from a green log. So the control case is measured on + # the SAME runner: this commit is the chart's pre-#88 fixture, the next one + # puts the mount back with the numbers written down. If this commit is what + # reaches main, the restore was lost and the install-test no longer walks the + # path #88 added. + # extraVolumes: + # - name: login-theme + # configMap: + # name: ci-keycloak-theme + # items: + # - key: theme.properties + # path: login/theme.properties + # extraVolumeMounts: + # - name: login-theme + # mountPath: /opt/keycloak/themes/ci-theme + # readOnly: true ingress: # Left ON although no ingress controller answers in CI: the point is that -- 2.54.0 From 8cd0c14e3fe94cd501a8ac2a03875c7d79e3e1bf Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 11:29:44 +0000 Subject: [PATCH 7/8] test(ci): control run 598 recorded (81s, 0 restarts) --- ci/keycloak/test-values.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index d3d7af1..aa66a25 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -186,7 +186,10 @@ keycloak: # three runs of PR #87 without it restarted none - too close to be dismissed # and not establishable from a green log. So the control case is measured on # the SAME runner: this commit is the chart's pre-#88 fixture, the next one - # puts the mount back with the numbers written down. If this commit is what + # puts the mount back with the numbers written down. + # Control runs so far: 598 -> 81s, 0 restarts (pod template hash 5844db56b6, + # i.e. byte-identical to the spec PR #87 measured). + # If this commit is what # reaches main, the restore was lost and the install-test no longer walks the # path #88 added. # extraVolumes: -- 2.54.0 From 6e196c26a569506e27d0f7638f3ad45736cd6580 Mon Sep 17 00:00:00 2001 From: Justin Klein Date: Tue, 6 Oct 2026 11:43:46 +0000 Subject: [PATCH 8/8] test(ci): restore the theme mount - the A/B shows the restart is pre-existing --- ci/keycloak/test-values.yaml | 93 ++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/ci/keycloak/test-values.yaml b/ci/keycloak/test-values.yaml index aa66a25..4d5f552 100644 --- a/ci/keycloak/test-values.yaml +++ b/ci/keycloak/test-values.yaml @@ -30,32 +30,39 @@ # first over the database would fail the run. Measured in Gitea PR #87: # 71.8s from `helm install` to both Ready, 0 restarts - three runs, all # three 72s and 0 restarts. -# THE RUNS OF PR #92 WERE NOT THAT, AND THE NUMBERS ARE RECORDED RATHER -# THAN SMOOTHED. Identical pod spec in all four - the later commits on that -# branch change comments only: -# run 585: 101s, ONE restart on one of the two pods -# run 590: 82s, 0 restarts -# run 593: 191s, 0 restarts -# run 595: 101s, ONE restart on one of the two pods -# Three things that are worth more than the average: the restart is -# INTERMITTENT on an unchanged spec (2 of 4), it is always exactly ONE on -# ONE pod, about 35s before that pod reports Ready - and it does not track -# the runner's speed, which was the first explanation and does not hold: -# the FASTEST (82s) and the SLOWEST (191s) run restarted nothing, and the -# two that restarted were both 101s. -# WHAT THE RESTART IS HAS NOT BEEN ESTABLISHED, and that is the lasting -# finding of the series: the harness dumps container logs only when a run -# FAILS (diagnose() hangs off the EXIT trap behind "RC -ne 0") and all four -# passed - A GREEN RUN CANNOT EXPLAIN A RESTART IT REPORTS. Answering it +# THAT WAS LUCK, OR A FAST WEEK, AND NOT AN INVARIANT - which took an A/B +# inside Gitea PR #92 to establish, because the first five runs there made +# it look like the new theme mount had destabilized the start: +# WITH the theme mount (spec of #88) +# run 585: 101s, ONE restart on one pod +# run 590: 82s, 0 +# run 593: 191s, 0 +# run 595: 101s, ONE +# run 597: 111s, ONE -> 3 of 5 +# WITHOUT it (the same branch, the pair commented out; pod template +# hash 5844db56b6, i.e. BYTE-IDENTICAL to the spec PR #87 measured) +# run 598: 81s, 0 +# run 599: 171s, ONE -> 1 of 2 +# So the restart happens WITHOUT the mount as well, and #88 is not its +# cause. It is pre-existing, intermittent behaviour of THIS chart's first +# install: always exactly one restart on one pod, 35-80s before that pod +# reports Ready. It does not track the runner's speed either - the fastest +# (81s) and the slowest (191s) run of the whole series restarted nothing. +# WHAT THE RESTART IS HAS NOT BEEN ESTABLISHED, and that is the finding +# worth keeping: the harness dumps container logs only when a run FAILS +# (diagnose() hangs off the EXIT trap behind "RC -ne 0") and all seven runs +# PASSED - A GREEN RUN CANNOT EXPLAIN A RESTART IT REPORTS. Answering it # means making the final-state step print the restart reasons whenever a -# counter is above zero; that is a change to ci/run-install-test.sh and it -# was deliberately NOT made inside #88. -# The counting stops here on purpose - every further commit adds a run, and -# a comment that chases the tally is wrong at the next push. What is worth -# carrying is the shape, not the tally: "0 restarts" is an observation, not -# an invariant of this chart, and a first install that serializes two -# replicas on a schema migration lock is a plausible place for one - which -# is a hypothesis, stated as one, because nothing here measured it. +# counter is above zero; that is a change to ci/run-install-test.sh with a +# CI-CD wiki obligation of its own, and it was deliberately NOT made inside +# #88. A first install that serializes two replicas on the Liquibase +# migration lock is a plausible place for it - stated as the hypothesis it +# is, because nothing here measured it. +# THE METHOD IS THE PART TO REUSE: a counter that moved in the same round +# as a change is not evidence against the change until the control case has +# run on the SAME machine. Three clean runs two days earlier were not that +# control, and treating them as one would have cost this PR its conclusion +# in both directions. # 5. A MOUNTED LOGIN THEME SURVIVES ALL OF THAT (Gitea #88) - the full # argument sits at keycloak.extraVolumes further down, including what it # still does not prove. @@ -181,28 +188,20 @@ keycloak: # the subpath hides nothing is a property of the mount path rather than a # measurement - and the kcLogoIdP line needs a browser plus a realm setting, # neither of which exists in CI. - # TEMPORARILY DISABLED FOR AN A/B MEASUREMENT, RESTORED BEFORE THIS PR IS - # MERGED. Three of five runs with the mount restarted one pod once, while the - # three runs of PR #87 without it restarted none - too close to be dismissed - # and not establishable from a green log. So the control case is measured on - # the SAME runner: this commit is the chart's pre-#88 fixture, the next one - # puts the mount back with the numbers written down. - # Control runs so far: 598 -> 81s, 0 restarts (pod template hash 5844db56b6, - # i.e. byte-identical to the spec PR #87 measured). - # If this commit is what - # reaches main, the restore was lost and the install-test no longer walks the - # path #88 added. - # extraVolumes: - # - name: login-theme - # configMap: - # name: ci-keycloak-theme - # items: - # - key: theme.properties - # path: login/theme.properties - # extraVolumeMounts: - # - name: login-theme - # mountPath: /opt/keycloak/themes/ci-theme - # readOnly: true + extraVolumes: + - name: login-theme + configMap: + name: ci-keycloak-theme + items: + - key: theme.properties + path: login/theme.properties + extraVolumeMounts: + - name: login-theme + # THE SUBPATH, AND THE WHOLE POINT OF THE EXAMPLE: a volume on + # /opt/keycloak/themes would hide the image's own three themes and break + # the login page while the pod stayed Ready. + mountPath: /opt/keycloak/themes/ci-theme + readOnly: true ingress: # Left ON although no ingress controller answers in CI: the point is that -- 2.54.0