Redo the semantic domain import until it finishes

database/init/update-semantic-domains.sh

@@ -1,4 +1,15 @@
 #! /usr/bin/bash
+# A partial import leaves the collections non-empty but incomplete, so record
+# completion here and only on success. Doing it here rather than in the caller
+# means a manual run also counts, and stops the next container start from
+# redoing the whole import.
+#
+# Stop at the first failure and report it: The Combine cannot be used without the
+# semantic domains, so the database's postStart hook restarts the container on a
+# non-zero exit rather than leave a database that looks healthy without them.
+set -eo pipefail
 
 mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang
 mongoimport -d CombineDatabase -c SemanticDomains /data/semantic-domains/nodes.json --mode=merge --upsertFields=id,guid,lang
+
+mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });"

deploy/helm/thecombine/charts/database/templates/database.yaml

@@ -56,8 +56,17 @@ spec:
                   - /bin/sh
                   - -c
                   - |
-                    exec > /data/db/postStart.log 2>&1
+                    log=/data/db/postStart.log
+                    # One block is appended per container start. Keeping the last
+                    # 200 lines keeps the recent starts and drops the rest, but it
+                    # does not respect block boundaries, so the oldest block that
+                    # survives can begin part way through.
+                    if [ -f "${log}" ]; then
+                      tail -n 200 "${log}" > "${log}.trim" && mv "${log}.trim" "${log}"
+                    fi
+                    exec >> "${log}" 2>&1
                     set -e
+                    echo "[postStart] $(date -Is) starting"
                     echo "[postStart] Waiting for mongod to accept connections"
                     attempts=0
                     until mongosh --quiet --host 127.0.0.1 --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; do
@@ -70,10 +79,31 @@ spec:
                     done
                     echo "[postStart] Ensuring replica set host"
                     mongosh --quiet --host 127.0.0.1 /opt/thecombine/00-replica-set.js || exit $?
-                    needs_semantic_import="$(mongosh --quiet --host 127.0.0.1 --eval "const combineDb = db.getSiblingDB('CombineDatabase'); const treeCount = combineDb.SemanticDomainTree.countDocuments({}); const domainCount = combineDb.SemanticDomains.countDocuments({}); print(treeCount === 0 || domainCount === 0 ? 'yes' : 'no');")"
-                    if [ "${needs_semantic_import}" = "yes" ]; then
-                      /bin/bash /opt/thecombine/update-semantic-domains.sh
+                    # Only a finished import is recorded, so an interrupted one is
+                    # redone. The record is read by exit status; anything short of a
+                    # completed import, including a query that fails, imports again,
+                    # which is a merge and so safe to repeat. Only stdout is discarded,
+                    # to leave any error mongosh reports in this log.
+                    import_done="quit(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 0 : 1)"
+                    if ! mongosh --quiet --host 127.0.0.1 --eval "${import_done}" > /dev/null; then
+                      echo "[postStart] Importing semantic domains"
+                      # The Combine cannot be used without the semantic domains, so a
+                      # failed import fails the hook, which restarts the container to
+                      # retry, rather than leaving a database that looks healthy.
+                      if ! /bin/bash /opt/thecombine/update-semantic-domains.sh; then
+                        echo "[postStart] Semantic domain import failed; restarting the container"
+                        exit 1
+                      fi
+                      # The script records the import too, which is what makes a manual
+                      # rerun count. Recording it here as well is an upsert either way,
+                      # and is what an image whose script predates the record needs.
+                      mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });"
                     fi
+                    # The kubelet does not probe a container until its postStart hook
+                    # returns, so this marker is written last: it cannot make the pod
+                    # ready any sooner, and everything above it has to succeed first.
+                    touch /tmp/replica-set-ready
+                    echo "[postStart] $(date -Is) done"
           env:
             - name: POD_IP
               valueFrom:
@@ -83,6 +113,21 @@ spec:
               value: "$(POD_IP):27017"
           ports:
             - containerPort: 27017
+          readinessProbe:
+            # /tmp/replica-set-ready is written at the end of the postStart hook,
+            # once mongod is a writable primary advertising this pod's IP and the
+            # semantic domains are in place. The pod IP is part of the replica set
+            # config and changes on every restart, so without this the Service can
+            # route the backend to a mongod it cannot use. A hook that fails, and
+            # so restarts the container, never writes the marker at all.
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - test -f /tmp/replica-set-ready
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 5
           resources:
             requests:
               cpu:  25m

docs/deploy/README.md

@@ -423,14 +423,40 @@ Notes:
 - When the `./setup_combine.py` script is used to install _The Combine_ on a NUC, it will install the fonts required for
   Arabic, English, French, Portuguese, and Spanish. If additional fonts will be required, call the `setup_combine.py`
   commands with the `--langs` option. Use the `--help` option to see the argument syntax.
-- The database pod has a `postStart` lifecycle hook that runs `update-semantic-domains.sh` automatically on every pod
-  start, but only imports data if the `SemanticDomainTree` or `SemanticDomains` collections are empty. If the Semantic
-  Domain data are updated, for example, adding a new language, then the script needs to be rerun manually:
+- The database pod has a `postStart` lifecycle hook that initializes the `rs0` replica set on every pod start and, if
+  the import has not already been recorded as complete, runs `update-semantic-domains.sh`. That script records its own
+  completion in `CombineDatabase.SemanticDomainImportStatus`, and only on success, so that a manual rerun counts as
+  well; the hook records it again afterwards, which is what an older image, whose script does not write the record,
+  needs. A count of the imported collections is not used, because an import that is interrupted part way through leaves
+  them non-empty but incomplete. If the import is interrupted, the next pod start redoes it. If the import fails
+  outright, the hook fails, and the kubelet restarts the container to retry; _The Combine_ cannot be used without the
+  semantic domains, so this is preferred over a database that looks healthy without them. Look in the `postStart` log,
+  below, to see why an import is failing.
+
+  The hook then writes `/tmp/replica-set-ready`, which is what the pod's readiness probe checks. Until then the pod is
+  kept out of the `database` Service endpoints, since the replica set advertises the pod's IP and the backend connects
+  with `?replicaSet=rs0`. Note that the kubelet does not probe a container until its `postStart` hook returns, so the
+  pod cannot become ready during the import no matter when the marker is written.
+
+  The completion record is in the database's persistent volume, so it outlives the pod that wrote it: once an import is
+  recorded, no later pod start imports again. An installation that imported before the record existed does not have one,
+  so its first pod start on this chart imports once more, and the pod stays out of the `database` Service for the few
+  minutes that takes. Whenever the Semantic Domain data change, the import has to be rerun manually. That is true both
+  of data added by hand, for example a new language, and of a release that ships an updated `tree.json` or `nodes.json`,
+  including one installed with `combinectl update`; neither is picked up on its own. A manual run refreshes the
+  completion record, so it is not redone on the next pod start:
 
   ```console
   kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh
   ```
 
+  The `postStart` hook appends its output to `/data/db/postStart.log`, which is on the database's persistent volume and
+  so survives restarts. Only the last 200 lines are kept, so the oldest entry in it may begin part way through:
+
+  ```console
+  kubectl -n thecombine exec deployment/database -- cat /data/db/postStart.log
+  ```
+
 ## Maintenance
 
 ### Maintenance Scripts for Kubernetes

installer/README.md

@@ -166,7 +166,7 @@ To run `combine-installer.run` with options, the option list must be started wit
 | clean           | Remove the previously saved environment (AWS Access Key, admin user info) and any previously saved timeout before performing the installation. |
 | restart         | Run the installation from the beginning; do not resume a previous installation. |
 | server          | Install _The Combine_ in a server environment so that _The Combine_ is always running by default. |
-| timeout TIMEOUT | Use a different timeout when installing. (Default: 5 minutes.) With slow internet, it is helpful to extend the timeout. See <https://pkg.go.dev/time#ParseDuration> for timeout formats. The value is used for the rest of the installation, including after a restart, so it does not need to be entered again. |
+| timeout TIMEOUT | Use a different timeout when installing. (Default: 5 minutes.) With slow internet, it is helpful to extend the timeout. See <https://pkg.go.dev/time#ParseDuration> for timeout formats. The value is used for the rest of the installation, including after a restart, so it does not need to be entered again. On a first installation the timeout also has to cover the semantic domain import, which takes several minutes; if it runs out, rerun the installer and it continues from where it stopped. |
 | uninstall       | Remove software installed by this script. |
 | update          | Update _The Combine_ to the version number provided. This skips installing/updating support software that was installed previously (e.g., the `combinectl` tool). |
 | version-number  | Specify a version to install instead of the current version. A version number will have the form `vn.n.n` where `n` represents an integer value, for example, `v1.20.0`. |