From 7fa1e800b122ed0e2abea5ad7269b03db1c9da93 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 21 Aug 2026 08:18:30 -0400 Subject: [PATCH 1/7] Guard the semantic domain import with a completion record The database pod's postStart hook imported the semantic domains when either collection was empty, so an import killed part way through left them non-empty but incomplete and was never redone. Record a finished import instead, and only when it finishes. Add a readiness probe on a marker the hook writes, so the pod stays out of the database Service until the replica set advertises the current pod IP, which changes on every restart and which the backend needs since it connects with ?replicaSet=rs0. The hook now appends to its log rather than truncating it, and stamps each start, so that earlier starts survive a restart. Co-Authored-By: Claude Opus 5 (1M context) --- database/init/update-semantic-domains.sh | 3 +++ .../charts/database/templates/database.yaml | 27 ++++++++++++++++--- docs/deploy/README.md | 22 ++++++++++++--- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/database/init/update-semantic-domains.sh b/database/init/update-semantic-domains.sh index 275bc7c89f..53f1324f55 100755 --- a/database/init/update-semantic-domains.sh +++ b/database/init/update-semantic-domains.sh @@ -1,4 +1,7 @@ #! /usr/bin/bash +# The caller records the import as complete only if this script succeeds, so a +# failed import must not be reported as success. +set -eo pipefail mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang mongoimport -d CombineDatabase -c SemanticDomains /data/semantic-domains/nodes.json --mode=merge --upsertFields=id,guid,lang diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index febca4a4af..184f6b541a 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -56,8 +56,9 @@ spec: - /bin/sh - -c - | - exec > /data/db/postStart.log 2>&1 + exec >> /data/db/postStart.log 2>&1 set -e + echo "[postStart] $(date -Is) starting" echo "[postStart] Waiting for mongod to accept connections" attempts=0 until mongosh --quiet --host 127.0.0.1 --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; do @@ -70,10 +71,17 @@ spec: done echo "[postStart] Ensuring replica set host" mongosh --quiet --host 127.0.0.1 /opt/thecombine/00-replica-set.js || exit $? - needs_semantic_import="$(mongosh --quiet --host 127.0.0.1 --eval "const combineDb = db.getSiblingDB('CombineDatabase'); const treeCount = combineDb.SemanticDomainTree.countDocuments({}); const domainCount = combineDb.SemanticDomains.countDocuments({}); print(treeCount === 0 || domainCount === 0 ? 'yes' : 'no');")" - if [ "${needs_semantic_import}" = "yes" ]; then + # mongod can serve the backend from here on, so let the readiness + # probe pass while the semantic domain import runs. + touch /tmp/replica-set-ready + # Only a finished import is recorded, so an interrupted one is redone. + import_state="$(mongosh --quiet --host 127.0.0.1 --eval "print(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 'done' : 'needed');")" + if [ "${import_state}" = "needed" ]; then + echo "[postStart] Importing semantic domains" /bin/bash /opt/thecombine/update-semantic-domains.sh + mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });" fi + echo "[postStart] $(date -Is) done" env: - name: POD_IP valueFrom: @@ -83,6 +91,19 @@ spec: value: "$(POD_IP):27017" ports: - containerPort: 27017 + readinessProbe: + # /tmp/replica-set-ready is written by the postStart hook once mongod is + # a writable primary advertising this pod's IP. The pod IP is part of + # the replica set config and changes on every restart, so without this + # the Service can route the backend to a mongod it cannot use. + exec: + command: + - /bin/sh + - -c + - test -f /tmp/replica-set-ready + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 resources: requests: cpu: 25m diff --git a/docs/deploy/README.md b/docs/deploy/README.md index ec9de372ff..048f03a734 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -423,14 +423,30 @@ Notes: - When the `./setup_combine.py` script is used to install _The Combine_ on a NUC, it will install the fonts required for Arabic, English, French, Portuguese, and Spanish. If additional fonts will be required, call the `setup_combine.py` commands with the `--langs` option. Use the `--help` option to see the argument syntax. -- The database pod has a `postStart` lifecycle hook that runs `update-semantic-domains.sh` automatically on every pod - start, but only imports data if the `SemanticDomainTree` or `SemanticDomains` collections are empty. If the Semantic - Domain data are updated, for example, adding a new language, then the script needs to be rerun manually: +- The database pod has a `postStart` lifecycle hook that initializes the `rs0` replica set on every pod start and, if + the import has not already been recorded as complete, runs `update-semantic-domains.sh`. The hook records completion + in `CombineDatabase.SemanticDomainImportStatus`; a count of the imported collections is not used, because an import + that is interrupted part way through leaves them non-empty but incomplete. If the import is interrupted, the next pod + start redoes it. + + The hook also writes `/tmp/replica-set-ready` once the replica set is aligned, which is what the pod's readiness probe + checks. Until then the pod is kept out of the `database` Service endpoints, since the replica set advertises the pod's + IP and the backend connects with `?replicaSet=rs0`. + + If the Semantic Domain data are updated, for example, adding a new language, then the import needs to be rerun + manually: ```console kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh ``` + The `postStart` hook appends its output to `/data/db/postStart.log`, which is on the database's persistent volume and + so survives restarts: + + ```console + kubectl -n thecombine exec deployment/database -- cat /data/db/postStart.log + ``` + ## Maintenance ### Maintenance Scripts for Kubernetes From 07998ab6890103240ccb907b82e69651668eba98 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 21 Aug 2026 08:18:31 -0400 Subject: [PATCH 2/7] Record the semantic domain import from the script that performs it The completion record was written by the postStart hook, so the documented manual rerun of update-semantic-domains.sh left it untouched and the next pod start redid the whole import. Write it from the script instead, after both imports succeed, so both paths agree. Also trim /data/db/postStart.log to the most recent starts. It is appended to on every container start and lives on the database's persistent volume, with nothing rotating it. Co-Authored-By: Claude Opus 5 (1M context) --- database/init/update-semantic-domains.sh | 8 ++++++-- .../charts/database/templates/database.yaml | 9 +++++++-- docs/deploy/README.md | 12 ++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/database/init/update-semantic-domains.sh b/database/init/update-semantic-domains.sh index 53f1324f55..482238dd5b 100755 --- a/database/init/update-semantic-domains.sh +++ b/database/init/update-semantic-domains.sh @@ -1,7 +1,11 @@ #! /usr/bin/bash -# The caller records the import as complete only if this script succeeds, so a -# failed import must not be reported as success. +# A partial import leaves the collections non-empty but incomplete, so record +# completion here and only on success. Doing it here rather than in the caller +# means a manual run also counts, and stops the next container start from +# redoing the whole import. set -eo pipefail mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang mongoimport -d CombineDatabase -c SemanticDomains /data/semantic-domains/nodes.json --mode=merge --upsertFields=id,guid,lang + +mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });" diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index 184f6b541a..88facd5b4d 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -56,7 +56,13 @@ spec: - /bin/sh - -c - | - exec >> /data/db/postStart.log 2>&1 + log=/data/db/postStart.log + # One block is appended per container start, so keep the most + # recent starts and drop the rest. + if [ -f "${log}" ]; then + tail -n 200 "${log}" > "${log}.trim" && mv "${log}.trim" "${log}" + fi + exec >> "${log}" 2>&1 set -e echo "[postStart] $(date -Is) starting" echo "[postStart] Waiting for mongod to accept connections" @@ -79,7 +85,6 @@ spec: if [ "${import_state}" = "needed" ]; then echo "[postStart] Importing semantic domains" /bin/bash /opt/thecombine/update-semantic-domains.sh - mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });" fi echo "[postStart] $(date -Is) done" env: diff --git a/docs/deploy/README.md b/docs/deploy/README.md index 048f03a734..2a8a7ef049 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -424,24 +424,24 @@ Notes: Arabic, English, French, Portuguese, and Spanish. If additional fonts will be required, call the `setup_combine.py` commands with the `--langs` option. Use the `--help` option to see the argument syntax. - The database pod has a `postStart` lifecycle hook that initializes the `rs0` replica set on every pod start and, if - the import has not already been recorded as complete, runs `update-semantic-domains.sh`. The hook records completion - in `CombineDatabase.SemanticDomainImportStatus`; a count of the imported collections is not used, because an import - that is interrupted part way through leaves them non-empty but incomplete. If the import is interrupted, the next pod - start redoes it. + the import has not already been recorded as complete, runs `update-semantic-domains.sh`. That script records its own + completion in `CombineDatabase.SemanticDomainImportStatus`, and only on success; a count of the imported collections + is not used, because an import that is interrupted part way through leaves them non-empty but incomplete. If the + import is interrupted, the next pod start redoes it. The hook also writes `/tmp/replica-set-ready` once the replica set is aligned, which is what the pod's readiness probe checks. Until then the pod is kept out of the `database` Service endpoints, since the replica set advertises the pod's IP and the backend connects with `?replicaSet=rs0`. If the Semantic Domain data are updated, for example, adding a new language, then the import needs to be rerun - manually: + manually. This also refreshes the completion record, so a manual import is not redone on the next pod start: ```console kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh ``` The `postStart` hook appends its output to `/data/db/postStart.log`, which is on the database's persistent volume and - so survives restarts: + so survives restarts. Only the most recent starts are kept: ```console kubectl -n thecombine exec deployment/database -- cat /data/db/postStart.log From 1a47a091e4c8cd8a422c7a1d79e0cb1c3bf70241 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 21 Aug 2026 08:18:31 -0400 Subject: [PATCH 3/7] Keep the database out of service until its setup finishes The readiness marker was written before the semantic domain import so that the probe could pass while the import ran, but the kubelet does not probe a container until its postStart hook returns, and the import runs inside that hook. The pod could not join the database Service any sooner, so write the marker last, where it also means that everything before it succeeded. Report a failed import in the postStart log and fail the hook explicitly, so the container restarts and retries: The Combine cannot be used without the semantic domains, and a database that serves without them looks healthy. Co-Authored-By: Claude Opus 5 (1M context) --- database/init/update-semantic-domains.sh | 4 +++ .../charts/database/templates/database.yaml | 25 +++++++++++++------ docs/deploy/README.md | 13 ++++++---- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/database/init/update-semantic-domains.sh b/database/init/update-semantic-domains.sh index 482238dd5b..c51f6bada1 100755 --- a/database/init/update-semantic-domains.sh +++ b/database/init/update-semantic-domains.sh @@ -3,6 +3,10 @@ # completion here and only on success. Doing it here rather than in the caller # means a manual run also counts, and stops the next container start from # redoing the whole import. +# +# Stop at the first failure and report it: The Combine cannot be used without the +# semantic domains, so the database's postStart hook restarts the container on a +# non-zero exit rather than leave a database that looks healthy without them. set -eo pipefail mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index 88facd5b4d..d0e92364b0 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -77,15 +77,22 @@ spec: done echo "[postStart] Ensuring replica set host" mongosh --quiet --host 127.0.0.1 /opt/thecombine/00-replica-set.js || exit $? - # mongod can serve the backend from here on, so let the readiness - # probe pass while the semantic domain import runs. - touch /tmp/replica-set-ready # Only a finished import is recorded, so an interrupted one is redone. import_state="$(mongosh --quiet --host 127.0.0.1 --eval "print(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 'done' : 'needed');")" if [ "${import_state}" = "needed" ]; then echo "[postStart] Importing semantic domains" - /bin/bash /opt/thecombine/update-semantic-domains.sh + # The Combine cannot be used without the semantic domains, so a + # failed import fails the hook, which restarts the container to + # retry, rather than leaving a database that looks healthy. + if ! /bin/bash /opt/thecombine/update-semantic-domains.sh; then + echo "[postStart] Semantic domain import failed; restarting the container" + exit 1 + fi fi + # The kubelet does not probe a container until its postStart hook + # returns, so this marker is written last: it cannot make the pod + # ready any sooner, and everything above it has to succeed first. + touch /tmp/replica-set-ready echo "[postStart] $(date -Is) done" env: - name: POD_IP @@ -97,10 +104,12 @@ spec: ports: - containerPort: 27017 readinessProbe: - # /tmp/replica-set-ready is written by the postStart hook once mongod is - # a writable primary advertising this pod's IP. The pod IP is part of - # the replica set config and changes on every restart, so without this - # the Service can route the backend to a mongod it cannot use. + # /tmp/replica-set-ready is written at the end of the postStart hook, + # once mongod is a writable primary advertising this pod's IP and the + # semantic domains are in place. The pod IP is part of the replica set + # config and changes on every restart, so without this the Service can + # route the backend to a mongod it cannot use. A hook that fails, and + # so restarts the container, never writes the marker at all. exec: command: - /bin/sh diff --git a/docs/deploy/README.md b/docs/deploy/README.md index 2a8a7ef049..083619ef08 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -427,11 +427,14 @@ Notes: the import has not already been recorded as complete, runs `update-semantic-domains.sh`. That script records its own completion in `CombineDatabase.SemanticDomainImportStatus`, and only on success; a count of the imported collections is not used, because an import that is interrupted part way through leaves them non-empty but incomplete. If the - import is interrupted, the next pod start redoes it. - - The hook also writes `/tmp/replica-set-ready` once the replica set is aligned, which is what the pod's readiness probe - checks. Until then the pod is kept out of the `database` Service endpoints, since the replica set advertises the pod's - IP and the backend connects with `?replicaSet=rs0`. + import is interrupted, the next pod start redoes it. If the import fails outright, the hook fails, and the kubelet + restarts the container to retry; _The Combine_ cannot be used without the semantic domains, so this is preferred over + a database that looks healthy without them. Look in the `postStart` log, below, to see why an import is failing. + + The hook then writes `/tmp/replica-set-ready`, which is what the pod's readiness probe checks. Until then the pod is + kept out of the `database` Service endpoints, since the replica set advertises the pod's IP and the backend connects + with `?replicaSet=rs0`. Note that the kubelet does not probe a container until its `postStart` hook returns, so the + pod cannot become ready during the import no matter when the marker is written. If the Semantic Domain data are updated, for example, adding a new language, then the import needs to be rerun manually. This also refreshes the completion record, so a manual import is not redone on the next pod start: From 2decf801d6ca04bb39764d9466ce71843c207a0e Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 21 Aug 2026 08:19:39 -0400 Subject: [PATCH 4/7] Correct what the database's comments and docs claim - The completion record outlives the pod that wrote it, so a release that ships updated semantic domain data needs the manual import too. - The postStart log is trimmed to 200 lines rather than to whole entries, so its oldest entry can begin part way through. Comments now use one space after a period rather than two. Co-Authored-By: Claude Opus 5 (1M context) --- database/init/update-semantic-domains.sh | 2 +- .../thecombine/charts/database/templates/database.yaml | 10 ++++++---- docs/deploy/README.md | 9 ++++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/database/init/update-semantic-domains.sh b/database/init/update-semantic-domains.sh index c51f6bada1..02afc8a546 100755 --- a/database/init/update-semantic-domains.sh +++ b/database/init/update-semantic-domains.sh @@ -1,6 +1,6 @@ #! /usr/bin/bash # A partial import leaves the collections non-empty but incomplete, so record -# completion here and only on success. Doing it here rather than in the caller +# completion here and only on success. Doing it here rather than in the caller # means a manual run also counts, and stops the next container start from # redoing the whole import. # diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index d0e92364b0..4bbf4c6c8d 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -57,8 +57,10 @@ spec: - -c - | log=/data/db/postStart.log - # One block is appended per container start, so keep the most - # recent starts and drop the rest. + # One block is appended per container start. Keeping the last + # 200 lines keeps the recent starts and drops the rest, but it + # does not respect block boundaries, so the oldest block that + # survives can begin part way through. if [ -f "${log}" ]; then tail -n 200 "${log}" > "${log}.trim" && mv "${log}.trim" "${log}" fi @@ -106,9 +108,9 @@ spec: readinessProbe: # /tmp/replica-set-ready is written at the end of the postStart hook, # once mongod is a writable primary advertising this pod's IP and the - # semantic domains are in place. The pod IP is part of the replica set + # semantic domains are in place. The pod IP is part of the replica set # config and changes on every restart, so without this the Service can - # route the backend to a mongod it cannot use. A hook that fails, and + # route the backend to a mongod it cannot use. A hook that fails, and # so restarts the container, never writes the marker at all. exec: command: diff --git a/docs/deploy/README.md b/docs/deploy/README.md index 083619ef08..60da610ac6 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -436,15 +436,18 @@ Notes: with `?replicaSet=rs0`. Note that the kubelet does not probe a container until its `postStart` hook returns, so the pod cannot become ready during the import no matter when the marker is written. - If the Semantic Domain data are updated, for example, adding a new language, then the import needs to be rerun - manually. This also refreshes the completion record, so a manual import is not redone on the next pod start: + The completion record is in the database's persistent volume, so it outlives the pod that wrote it: once an import is + recorded, no later pod start imports again. Whenever the Semantic Domain data change, the import has to be rerun + manually. That is true both of data added by hand, for example a new language, and of a release that ships an updated + `tree.json` or `nodes.json`, including one installed with `combinectl update`; neither is picked up on its own. A + manual run refreshes the completion record, so it is not redone on the next pod start: ```console kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh ``` The `postStart` hook appends its output to `/data/db/postStart.log`, which is on the database's persistent volume and - so survives restarts. Only the most recent starts are kept: + so survives restarts. Only the last 200 lines are kept, so the oldest entry in it may begin part way through: ```console kubectl -n thecombine exec deployment/database -- cat /data/db/postStart.log From 70e73e66df4f8ed94fecc72cad92453f7a39c4ba Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 21 Aug 2026 08:19:39 -0400 Subject: [PATCH 5/7] Read the semantic domain import record by exit status The postStart hook captured mongosh's output and imported only when it was exactly "needed", so a stray line on stdout left the pod ready with no semantic domains, and a failed query aborted the hook through set -e with nothing in the log to say why. Branch on mongosh's exit status instead: anything short of a completed import runs the import, which is a merge and safe to repeat, and only stdout is discarded so an error still reaches the postStart log. Co-Authored-By: Claude Opus 5 (1M context) --- .../thecombine/charts/database/templates/database.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index 4bbf4c6c8d..f6aeae3932 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -79,9 +79,13 @@ spec: done echo "[postStart] Ensuring replica set host" mongosh --quiet --host 127.0.0.1 /opt/thecombine/00-replica-set.js || exit $? - # Only a finished import is recorded, so an interrupted one is redone. - import_state="$(mongosh --quiet --host 127.0.0.1 --eval "print(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 'done' : 'needed');")" - if [ "${import_state}" = "needed" ]; then + # Only a finished import is recorded, so an interrupted one is + # redone. The record is read by exit status; anything short of a + # completed import, including a query that fails, imports again, + # which is a merge and so safe to repeat. Only stdout is discarded, + # to leave any error mongosh reports in this log. + import_done="quit(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 0 : 1)" + if ! mongosh --quiet --host 127.0.0.1 --eval "${import_done}" > /dev/null; then echo "[postStart] Importing semantic domains" # The Combine cannot be used without the semantic domains, so a # failed import fails the hook, which restarts the container to From 6debcbbb4f2f2710baec1c87290e473b697666f3 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 21 Aug 2026 08:19:40 -0400 Subject: [PATCH 6/7] Note what an install with no import record does on its first start The completion record is in the database's persistent volume, so it outlives the pod that wrote it. An installation that imported before the record existed has none, so its first pod start on this chart imports once more and stays out of the database Service until it is done. A timeout given to the installer therefore has to cover that import, since the database is not available until its postStart hook has finished it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/deploy/README.md | 10 ++++++---- installer/README.md | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/deploy/README.md b/docs/deploy/README.md index 60da610ac6..bc5c621a83 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -437,10 +437,12 @@ Notes: pod cannot become ready during the import no matter when the marker is written. The completion record is in the database's persistent volume, so it outlives the pod that wrote it: once an import is - recorded, no later pod start imports again. Whenever the Semantic Domain data change, the import has to be rerun - manually. That is true both of data added by hand, for example a new language, and of a release that ships an updated - `tree.json` or `nodes.json`, including one installed with `combinectl update`; neither is picked up on its own. A - manual run refreshes the completion record, so it is not redone on the next pod start: + recorded, no later pod start imports again. An installation that imported before the record existed does not have one, + so its first pod start on this chart imports once more, and the pod stays out of the `database` Service for the few + minutes that takes. Whenever the Semantic Domain data change, the import has to be rerun manually. That is true both + of data added by hand, for example a new language, and of a release that ships an updated `tree.json` or `nodes.json`, + including one installed with `combinectl update`; neither is picked up on its own. A manual run refreshes the + completion record, so it is not redone on the next pod start: ```console kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh diff --git a/installer/README.md b/installer/README.md index 1f676d9497..d58fb1b956 100644 --- a/installer/README.md +++ b/installer/README.md @@ -166,7 +166,7 @@ To run `combine-installer.run` with options, the option list must be started wit | clean | Remove the previously saved environment (AWS Access Key, admin user info) and any previously saved timeout before performing the installation. | | restart | Run the installation from the beginning; do not resume a previous installation. | | server | Install _The Combine_ in a server environment so that _The Combine_ is always running by default. | -| timeout TIMEOUT | Use a different timeout when installing. (Default: 5 minutes.) With slow internet, it is helpful to extend the timeout. See for timeout formats. The value is used for the rest of the installation, including after a restart, so it does not need to be entered again. | +| timeout TIMEOUT | Use a different timeout when installing. (Default: 5 minutes.) With slow internet, it is helpful to extend the timeout. See for timeout formats. The value is used for the rest of the installation, including after a restart, so it does not need to be entered again. On a first installation the timeout also has to cover the semantic domain import, which takes several minutes; if it runs out, rerun the installer and it continues from where it stopped. | | uninstall | Remove software installed by this script. | | update | Update _The Combine_ to the version number provided. This skips installing/updating support software that was installed previously (e.g., the `combinectl` tool). | | version-number | Specify a version to install instead of the current version. A version number will have the form `vn.n.n` where `n` represents an integer value, for example, `v1.20.0`. | From 7191cb25d6bcaaad536a41faddd60a3cfbe29db1 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 21 Aug 2026 10:41:00 -0400 Subject: [PATCH 7/7] Record the semantic domain import from the hook as well The record is written by update-semantic-domains.sh, which is in the database image, while the hook that reads it is in the chart, which ships in the installer. An installer built from master before the next release pairs this chart with the previous release's image, whose script does not write the record, so nothing ever would: the import would be redone on every start, and the installer's wait for the record would never pass. Record it from the hook too, after the script succeeds. Both writes are the same upsert, and each covers a case the other does not: the script's is what makes a manual rerun count, the hook's is what makes an older image work. Co-Authored-By: Claude Opus 5 (1M context) --- .../charts/database/templates/database.yaml | 4 ++++ docs/deploy/README.md | 12 +++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index f6aeae3932..b70ed458a4 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -94,6 +94,10 @@ spec: echo "[postStart] Semantic domain import failed; restarting the container" exit 1 fi + # The script records the import too, which is what makes a manual + # rerun count. Recording it here as well is an upsert either way, + # and is what an image whose script predates the record needs. + mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });" fi # The kubelet does not probe a container until its postStart hook # returns, so this marker is written last: it cannot make the pod diff --git a/docs/deploy/README.md b/docs/deploy/README.md index bc5c621a83..23f962d523 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -425,11 +425,13 @@ Notes: commands with the `--langs` option. Use the `--help` option to see the argument syntax. - The database pod has a `postStart` lifecycle hook that initializes the `rs0` replica set on every pod start and, if the import has not already been recorded as complete, runs `update-semantic-domains.sh`. That script records its own - completion in `CombineDatabase.SemanticDomainImportStatus`, and only on success; a count of the imported collections - is not used, because an import that is interrupted part way through leaves them non-empty but incomplete. If the - import is interrupted, the next pod start redoes it. If the import fails outright, the hook fails, and the kubelet - restarts the container to retry; _The Combine_ cannot be used without the semantic domains, so this is preferred over - a database that looks healthy without them. Look in the `postStart` log, below, to see why an import is failing. + completion in `CombineDatabase.SemanticDomainImportStatus`, and only on success, so that a manual rerun counts as + well; the hook records it again afterwards, which is what an older image, whose script does not write the record, + needs. A count of the imported collections is not used, because an import that is interrupted part way through leaves + them non-empty but incomplete. If the import is interrupted, the next pod start redoes it. If the import fails + outright, the hook fails, and the kubelet restarts the container to retry; _The Combine_ cannot be used without the + semantic domains, so this is preferred over a database that looks healthy without them. Look in the `postStart` log, + below, to see why an import is failing. The hook then writes `/tmp/replica-set-ready`, which is what the pod's readiness probe checks. Until then the pod is kept out of the `database` Service endpoints, since the replica set advertises the pod's IP and the backend connects