diff --git a/database/init/update-semantic-domains.sh b/database/init/update-semantic-domains.sh index 275bc7c89f..02afc8a546 100755 --- a/database/init/update-semantic-domains.sh +++ b/database/init/update-semantic-domains.sh @@ -1,4 +1,15 @@ #! /usr/bin/bash +# A partial import leaves the collections non-empty but incomplete, so record +# completion here and only on success. Doing it here rather than in the caller +# means a manual run also counts, and stops the next container start from +# redoing the whole import. +# +# Stop at the first failure and report it: The Combine cannot be used without the +# semantic domains, so the database's postStart hook restarts the container on a +# non-zero exit rather than leave a database that looks healthy without them. +set -eo pipefail mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang mongoimport -d CombineDatabase -c SemanticDomains /data/semantic-domains/nodes.json --mode=merge --upsertFields=id,guid,lang + +mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });" diff --git a/deploy/ansible/roles/support_tools/files/combinectl.sh b/deploy/ansible/roles/support_tools/files/combinectl.sh index b1bb7eae34..bb5f0c8f75 100755 --- a/deploy/ansible/roles/support_tools/files/combinectl.sh +++ b/deploy/ansible/roles/support_tools/files/combinectl.sh @@ -31,9 +31,8 @@ usage () { .EOM } -# Get the name of the first wifi interface. In general, -# this script assumes that there is a single WiFi interface -# installed. +# Get the name of the first wifi interface. In general, this script assumes +# that there is a single WiFi interface installed. get-wifi-if () { IFS=$'\n' WIFI_DEVICES=( $(nmcli d | grep "^wl") ) if [[ ${#WIFI_DEVICES[@]} -gt 0 ]] ; then @@ -73,7 +72,133 @@ combine-cert () { echo $CERT_DATA | base64 -d | openssl x509 -enddate -noout| sed -e "s/^notAfter=/Web certificate expires at /" } -# Start The Combine services +# Report whether the Kubernetes API is serving requests. The k3s unit becomes +# active before the API is up, so an active unit alone is not enough. +cluster-ready () { + kubectl get --raw='/readyz' --request-timeout=10s > /dev/null 2>&1 +} + +# Wait up to two minutes for the Kubernetes API to serve requests. +wait-for-cluster () { + ATTEMPTS=0 + until cluster-ready ; do + ATTEMPTS=$((ATTEMPTS + 1)) + if [[ ${ATTEMPTS} -ge 60 ]] ; then + return 1 + fi + sleep 2 + done + return 0 +} + +# Print "name requested available" for every deployment in the namespace. +# availableReplicas is absent, rather than 0, when none are available. +combine-deployments () { + kubectl -n thecombine get deployments 2> /dev/null \ + -o 'jsonpath={range .items[*]}{.metadata.name} {.spec.replicas} {.status.availableReplicas}{"\n"}{end}' +} + +# Restore the deployment replica counts saved by stop-combine-deployments. The +# counts live in a local file, so fall back to one replica each when it is +# missing: k3s can be started without combinectl, which would otherwise leave +# the deployments scaled to zero with nothing to bring them back. +# +# Returns non-zero when a deployment that should be running is not, so that a +# failed start is not reported as a successful one. +start-combine-deployments () { + if ! wait-for-cluster ; then + echo "The cluster is not responding; run \"combinectl start\" again." >&2 + return 1 + fi + DEPLOY_STATUS=$(combine-deployments) + if [[ -z ${DEPLOY_STATUS} ]] ; then + # Nothing is installed, so there is nothing to start; combine-status is + # where an empty namespace is reported. + return 0 + fi + if [ -f "${CACHED_REPLICAS}" ] ; then + CACHE_FILE="${CACHED_REPLICAS}" + else + CACHE_FILE=/dev/null + fi + # List every deployment that is scaled down, with its saved count. The saved + # counts are reconciled with the cluster rather than replayed as they are: a + # cached deployment that no longer exists, for example one renamed by a chart + # update, would otherwise fail to scale on every start, and its failure would + # keep the stale file, and any deployment the file omits, forever. + REPLICA_LIST=$(awk ' + NR == FNR { if ($2 == 0) { stopped[$1] = 1 } ; next } + $1 in stopped && $2 > 0 { print $1, $2 ; delete stopped[$1] } + END { for (name in stopped) { print name, 1 } } + ' <(printf '%s\n' "${DEPLOY_STATUS}") "${CACHE_FILE}") + if [[ -z ${REPLICA_LIST} ]] ; then + # Nothing is scaled down, so any saved counts no longer apply. + rm -f "${CACHED_REPLICAS}" + return 0 + fi + if [[ ${CACHE_FILE} == /dev/null ]] ; then + echo "No saved replica counts; starting one replica of each deployment." + fi + echo "Starting The Combine deployments." + RESTORE_FAILED=0 + while read -r DEPLOYMENT REPLICAS ; do + if [[ -z ${DEPLOYMENT} || -z ${REPLICAS} ]] ; then + continue + fi + if ! kubectl -n thecombine scale "deployment/${DEPLOYMENT}" --replicas="${REPLICAS}" > /dev/null ; then + echo "Could not start deployment/${DEPLOYMENT}." >&2 + RESTORE_FAILED=1 + fi + done <<< "${REPLICA_LIST}" + # Keep the counts for the next attempt if any deployment was not restored. + if [[ ${RESTORE_FAILED} -eq 0 ]] ; then + rm -f "${CACHED_REPLICAS}" + fi + return ${RESTORE_FAILED} +} + +# Scale The Combine deployments to zero and wait for their pods to exit. The +# k3s service is patched to KillMode=mixed, so stopping it SIGKILLs whatever is +# still running, which can be the database part way through its startup setup. +# +# Returns non-zero if the cluster could not be reached, so that the caller leaves +# k3s running. A pod still terminating after two minutes only warns, so that one +# stuck pod cannot leave The Combine with no way to stop. +stop-combine-deployments () { + # An unreachable Kubernetes API looks exactly like an empty namespace, so wait + # for it: a stop issued while the cluster is still coming up must not be read + # as "nothing is running here." + if ! wait-for-cluster ; then + echo "The cluster is not responding, so The Combine was not stopped." >&2 + echo "Wait a minute, then run \"combinectl stop\" again." >&2 + return 1 + fi + DEPLOY_STATUS=$(combine-deployments) + if [[ -z ${DEPLOY_STATUS} ]] ; then + return 0 + fi + # Only record the deployments that are running, so that stopping The Combine + # when it's already stopped doesn't lose the counts. + REPLICA_LIST=$(awk '$2 > 0 { print $1, $2 }' <<< "${DEPLOY_STATUS}") + if [[ -n ${REPLICA_LIST} ]] ; then + echo "${REPLICA_LIST}" > "${CACHED_REPLICAS}" + fi + echo "Stopping The Combine deployments." + kubectl -n thecombine scale deployment --all --replicas=0 > /dev/null + # A selector-based "kubectl wait" fails immediately when nothing matches it, + # so only wait when there are pods left to wait for. + if [[ -n $(kubectl -n thecombine get pods -l combine-component --no-headers 2> /dev/null) ]] ; then + if ! kubectl -n thecombine wait --for=delete pod -l combine-component \ + --timeout=2m > /dev/null 2>&1 ; then + echo "The Combine did not stop within 2 minutes; stopping anyway." >&2 + fi + fi + return 0 +} + +# Start The Combine services. The status of the last command is the status of +# the function, so this returns non-zero if the deployments were not started, +# which matches combine-stop. combine-start () { echo "Starting The Combine." if ! systemctl is-active --quiet create_ap ; then @@ -84,13 +209,19 @@ combine-start () { if ! systemctl is-active --quiet k3s ; then sudo systemctl start k3s fi + start-combine-deployments } -# Stop The Combine services and restore the WiFI -# connection if needed. +# Stop The Combine services and restore the WiFi connection if needed. Returns +# non-zero if The Combine is still running. combine-stop () { echo "Stopping The Combine." if systemctl is-active --quiet k3s ; then + # Stopping k3s SIGKILLs the containers, so only do it once the deployments + # have shut down; leave everything running otherwise. + if ! stop-combine-deployments ; then + return 1 + fi sudo systemctl stop k3s fi if systemctl is-active --quiet create_ap ; then @@ -98,28 +229,88 @@ combine-stop () { restore-wifi-connection sudo systemctl restart systemd-resolved fi + return 0 } -# Print the status of The Combine services. If the combine is -# "up" then also print that status of the deployments in -# "thecombine" namespace. +# Print the status of The Combine services. When the cluster is up, also +# distinguish between The Combine being uninstalled, incompletely installed, +# scaled down, partly scaled down, still starting, and fully running, then print +# the status of the deployments in the "thecombine" namespace. +# +# Always exits 0; install-combine.sh calls this under "set -e". combine-status () { if systemctl is-active --quiet create_ap ; then echo "WiFi hotspot is Running." else echo "WiFi hotspot is Stopped." fi - if systemctl is-active --quiet k3s ; then - echo "The Combine is Running." - kubectl -n thecombine get deployments - else + + if ! systemctl is-active --quiet k3s ; then echo "The Combine is Stopped." + return 0 + fi + + if ! cluster-ready ; then + echo "The Combine is Starting; the Kubernetes cluster is not ready yet." + echo "Wait a minute, then run \"combinectl status\" again." + return 0 + fi + + DEPLOY_STATUS=$(combine-deployments) + if [[ -z ${DEPLOY_STATUS} ]] ; then + echo "The Combine is Not Installed; the Kubernetes cluster is running, but" + echo "the \"thecombine\" namespace has no deployments." + echo "Download and run the install package to install The Combine." + return 0 + fi + + MISSING=() + for DEPLOYMENT in "${COMBINE_DEPLOYMENTS[@]}" ; do + if ! grep -q "^${DEPLOYMENT} " <<< "${DEPLOY_STATUS}" ; then + MISSING+=( "${DEPLOYMENT}" ) + fi + done + + # Total requested replicas, to tell a scaled down Combine from a running one, + # the deployments that ask for no replicas at all, and the ones that do not + # have all of the replicas they asked for. A deployment scaled to zero has + # every replica it asked for, so it has to be counted separately from those. + REQUESTED=0 + STOPPED=() + PENDING=() + while read -r NAME WANT HAVE ; do + if [[ -z ${NAME} ]] ; then + continue + fi + REQUESTED=$(( REQUESTED + ${WANT:-0} )) + if [[ ${WANT:-0} -eq 0 ]] ; then + STOPPED+=( "${NAME}" ) + elif [[ ${HAVE:-0} -lt ${WANT:-0} ]] ; then + PENDING+=( "${NAME}" ) + fi + done <<< "${DEPLOY_STATUS}" + + if [[ ${#MISSING[@]} -gt 0 ]] ; then + echo "The Combine is Incomplete; missing deployment(s): ${MISSING[*]}." + echo "Download and run the install package to repair the installation." + elif [[ ${REQUESTED} -eq 0 ]] ; then + echo "The Combine is Stopped; the cluster is up but its services are" + echo "scaled down. Run \"combinectl start\" to start them." + elif [[ ${#STOPPED[@]} -gt 0 ]] ; then + echo "The Combine is Partly Stopped; scaled down deployment(s): ${STOPPED[*]}." + echo "Run \"combinectl start\" to start them." + elif [[ ${#PENDING[@]} -gt 0 ]] ; then + echo "The Combine is Starting; waiting for: ${PENDING[*]}." + else + echo "The Combine is Running." fi + kubectl -n thecombine get deployments + return 0 } -# Update the image used in each of the deployments in The Combine. This -# is akin to our current update process for Production and QA servers. It -# does *not* update any configuration files or secrets. +# Update the image used in each of the deployments in The Combine. This is akin +# to our current update process for Production and QA servers. It does *not* +# update any configuration files or secrets. combine-update () { echo "Updating The Combine to $1" IMAGE_TAG=$1 @@ -155,11 +346,15 @@ combine-wifi-set-password () { } # Main script entrypoint +# The deployments that make up The Combine, used to detect an installation that +# is missing components. Matches the list in install-combine.sh. +COMBINE_DEPLOYMENTS=(backend database frontend maintenance) WIFI_IF=$(get-wifi-if) WIFI_CONFIG=/etc/create_ap/create_ap.conf export KUBECONFIG=${HOME}/.kube/config COMBINE_CONFIG=${HOME}/.config/combine CACHED_WIFI_CONN=${COMBINE_CONFIG}/wifi-connection.txt +CACHED_REPLICAS=${COMBINE_CONFIG}/deployment-replicas.txt # Make sure config directory exists mkdir -p "${COMBINE_CONFIG}" diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index febca4a4af..f6aeae3932 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -56,8 +56,17 @@ spec: - /bin/sh - -c - | - exec > /data/db/postStart.log 2>&1 + log=/data/db/postStart.log + # One block is appended per container start. Keeping the last + # 200 lines keeps the recent starts and drops the rest, but it + # does not respect block boundaries, so the oldest block that + # survives can begin part way through. + if [ -f "${log}" ]; then + tail -n 200 "${log}" > "${log}.trim" && mv "${log}.trim" "${log}" + fi + exec >> "${log}" 2>&1 set -e + echo "[postStart] $(date -Is) starting" echo "[postStart] Waiting for mongod to accept connections" attempts=0 until mongosh --quiet --host 127.0.0.1 --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; do @@ -70,10 +79,27 @@ spec: done echo "[postStart] Ensuring replica set host" mongosh --quiet --host 127.0.0.1 /opt/thecombine/00-replica-set.js || exit $? - needs_semantic_import="$(mongosh --quiet --host 127.0.0.1 --eval "const combineDb = db.getSiblingDB('CombineDatabase'); const treeCount = combineDb.SemanticDomainTree.countDocuments({}); const domainCount = combineDb.SemanticDomains.countDocuments({}); print(treeCount === 0 || domainCount === 0 ? 'yes' : 'no');")" - if [ "${needs_semantic_import}" = "yes" ]; then - /bin/bash /opt/thecombine/update-semantic-domains.sh + # Only a finished import is recorded, so an interrupted one is + # redone. The record is read by exit status; anything short of a + # completed import, including a query that fails, imports again, + # which is a merge and so safe to repeat. Only stdout is discarded, + # to leave any error mongosh reports in this log. + import_done="quit(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 0 : 1)" + if ! mongosh --quiet --host 127.0.0.1 --eval "${import_done}" > /dev/null; then + echo "[postStart] Importing semantic domains" + # The Combine cannot be used without the semantic domains, so a + # failed import fails the hook, which restarts the container to + # retry, rather than leaving a database that looks healthy. + if ! /bin/bash /opt/thecombine/update-semantic-domains.sh; then + echo "[postStart] Semantic domain import failed; restarting the container" + exit 1 + fi fi + # The kubelet does not probe a container until its postStart hook + # returns, so this marker is written last: it cannot make the pod + # ready any sooner, and everything above it has to succeed first. + touch /tmp/replica-set-ready + echo "[postStart] $(date -Is) done" env: - name: POD_IP valueFrom: @@ -83,6 +109,21 @@ spec: value: "$(POD_IP):27017" ports: - containerPort: 27017 + readinessProbe: + # /tmp/replica-set-ready is written at the end of the postStart hook, + # once mongod is a writable primary advertising this pod's IP and the + # semantic domains are in place. The pod IP is part of the replica set + # config and changes on every restart, so without this the Service can + # route the backend to a mongod it cannot use. A hook that fails, and + # so restarts the container, never writes the marker at all. + exec: + command: + - /bin/sh + - -c + - test -f /tmp/replica-set-ready + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 resources: requests: cpu: 25m diff --git a/deploy/helm/thecombine/charts/frontend/templates/deployment-frontend.yaml b/deploy/helm/thecombine/charts/frontend/templates/deployment-frontend.yaml index cbb7845f43..a1adfc9d43 100644 --- a/deploy/helm/thecombine/charts/frontend/templates/deployment-frontend.yaml +++ b/deploy/helm/thecombine/charts/frontend/templates/deployment-frontend.yaml @@ -67,6 +67,13 @@ spec: ports: - containerPort: 80 - containerPort: 443 + readinessProbe: + # The entrypoint waits for cluster DNS to resolve the backend before + # starting nginx, so a running container is not yet a serving one. + tcpSocket: + port: 80 + initialDelaySeconds: 5 + periodSeconds: 5 resources: requests: cpu: 1m diff --git a/deploy/scripts/install-combine.sh b/deploy/scripts/install-combine.sh index ae6a3a0c64..e8d43ad9af 100755 --- a/deploy/scripts/install-combine.sh +++ b/deploy/scripts/install-combine.sh @@ -13,6 +13,10 @@ set -eo pipefail # - single-step - run the next "step" in the installation process and stop. # - start-at - start at the step named and run to completion. # +# The WAIT_TIMEOUT_SECONDS environment variable overrides how long the "Wait-for-combine" +# step waits for the deployments to become available. The check that follows it has a +# short budget of its own, since it normally passes at once. +# ######################################################################################### # Warning and Error reporting functions @@ -28,9 +32,9 @@ error () { exit 1 } -# Set the environment variables that are required by The Combine. -# In addition, the values are stored in a file so that they do not -# need to be re-entered on subsequent installations. +# Set the environment variables that are required by The Combine. In addition, +# the values are stored in a file so that they do not need to be re-entered on +# subsequent installations. set-combine-env () { if [ ! -f "${CONFIG_DIR}/env" ] ; then # Generate JWT Secret Key @@ -97,9 +101,8 @@ install-kubernetes () { ansible-playbook playbook_desktop_setup.yml -K ${EXTRA_VARS} $(((DEBUG == 1)) && echo "-vv") } -# Set the KUBECONFIG environment variable so that the cluster can -# be reached by the installation scripts. It also starts the k3s -# service if it is not already running. +# Set the KUBECONFIG env var so the cluster can be reached by the installation +# scripts. It also starts the k3s service if it is not already running. set-k3s-env () { ##### # Setup kubectl configuration file @@ -154,25 +157,75 @@ install-the-combine () { deactivate } -# Wait until all The Combine deployments are "Running" +# Start a wait of $1 seconds, keeping the budget for check-wait-deadline to report. +start-wait () { + WAIT_BUDGET=$1 + WAIT_DEADLINE=$(( SECONDS + WAIT_BUDGET )) +} + +# Exit if the deadline for the current wait has passed, printing the state of +# the pods so that the user has somewhere to start looking. $1 names what is +# still being waited for and $2 optionally replaces the default error hint. +check-wait-deadline () { + if (( SECONDS < WAIT_DEADLINE )) ; then + return 0 + fi + echo "Current state of the pods in the 'thecombine' namespace:" >&2 + kubectl -n thecombine get pods --request-timeout=10s >&2 || true + ERROR_HINT="${2:-Rerun the installer to resume waiting; nothing needs to be undone.}" + # The deadline covers the whole wait, so report what it was still waiting for + # rather than implying that $1 alone had the full timeout. + error "Timed out after ${WAIT_BUDGET}s; still waiting for $1." +} + +# Wait until all The Combine deployments are available. One deadline covers all +# of them, so the database, which is not available until its postStart hook has +# imported the semantic domains, leaves the rest of them less time. wait-for-combine () { - # Wait for all The Combine deployments to be up - while true ; do - combine_status=`kubectl -n thecombine get deployments` - # Assert The Combine is up; if any components are not up, set it to false - combine_up=true - for deployment in frontend backend database maintenance ; do - deployment_status=$(echo ${combine_status} | grep "${deployment}" | sed "s/^.*\([0-9]\)\/1.*/\1/") - if [ "$deployment_status" == "0" ] ; then - combine_up=false - break - fi + set-k3s-env + start-wait "${WAIT_TIMEOUT_SECONDS}" + # The database is checked first because everything else depends on it and it + # is the slowest to come up on a first install. + for deployment in database backend frontend maintenance ; do + echo "Waiting for deployment/${deployment}." + until kubectl -n thecombine get deployment/${deployment} > /dev/null 2>&1 ; do + check-wait-deadline "deployment/${deployment}" + sleep 5 done - if [ ${combine_up} != true ] ; then + # "kubectl wait" returns at once, rather than blocking for its timeout, when + # the API is unreachable or the deployment is gone, so pace the retries. + until kubectl -n thecombine wait --for=condition=Available \ + --timeout=1m deployment/${deployment} > /dev/null 2>&1 ; do + check-wait-deadline "deployment/${deployment}" + echo " still waiting for deployment/${deployment}." sleep 5 - else - break - fi + done + done +} + +# Check that the database has recorded a completed semantic domain import, which +# is redone on the next pod start if it was interrupted, so that the shutdown +# below does not silently cost the user another import. +# +# The import runs from the database's postStart hook, and the kubelet does not +# report a container ready until its hook returns, so wait-for-combine has +# already waited the import out and this normally passes on its first check. It +# is here for what that wait cannot see: a database pod that an install left +# running, and so never ran the hook, with no import recorded. Nothing will write +# the record then, so the wait is short and ends in the hint below. +wait-for-semantic-domains () { + echo "Waiting for the semantic domain import." + start-wait "${IMPORT_CHECK_TIMEOUT_SECONDS}" + import_done="quit(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 0 : 1)" + # Rerunning the installer does not restart the database pod, so it does not + # start an import that never ran; point at the manual import instead. + import_hint="Import the semantic domains manually with: kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh" + # Each check starts a mongosh inside the database container, competing with the + # import it is waiting on, so check infrequently. + until kubectl -n thecombine exec deployment/database -- \ + mongosh --quiet --host 127.0.0.1 --eval "${import_done}" > /dev/null 2>&1 ; do + check-wait-deadline "the semantic domain import" "${import_hint}" + sleep 30 done } @@ -236,9 +289,14 @@ SINGLE_STEP=0 IS_SERVER=0 DEBUG=0 ERROR_HINT="" -# Only a timeout given as an option is checked for a valid format, so ignore any -# value that happens to be set in the environment. +# Only a timeout given as an option is checked for a valid format, so ignore +# any value that happens to be set in the environment. HELM_TIMEOUT="" +# Maximum time to wait for each stage of The Combine to come up. A first install +# pulls several images and imports the semantic domains, so be generous. +WAIT_TIMEOUT_SECONDS=${WAIT_TIMEOUT_SECONDS:-3600} +# The deployments wait already covers the import, so this only covers one slow call. +IMPORT_CHECK_TIMEOUT_SECONDS=120 # See if we need to continue from a previous install STATE_FILE=${CONFIG_DIR}/install-state @@ -309,6 +367,10 @@ fi SETUP_OPTS="" if [ "${STATE}" != "Uninstall-combine" ] ; then + # Prevent silently bad values: non-number (reads as 0); leading zero (octal). + if [[ ! ${WAIT_TIMEOUT_SECONDS} =~ ^[1-9][0-9]*$ ]] ; then + error "Invalid WAIT_TIMEOUT_SECONDS, '${WAIT_TIMEOUT_SECONDS}'; it must be a whole number greater than zero." + fi # Every helm command runs after the restart that the Pre-reqs step usually # requires, so record a timeout and reuse it until the install finishes. if [ -z "${HELM_TIMEOUT}" ] ; then @@ -374,14 +436,21 @@ while [ "$STATE" != "Done" ] ; do echo "This may take some time depending on your Internet connection." echo "Press Ctrl-C to interrupt." wait-for-combine + wait-for-semantic-domains echo "The Combine was successfully setup!" next-state "Shutdown-combine" + if [ "$SINGLE_STEP" == "1" ] ; then + STATE=Done + fi ;; Shutdown-combine) # If not being installed as a server, if [[ $IS_SERVER != 1 ]] ; then - # Shut down The Combine services - combinectl stop + # Shut down The Combine services. combinectl leaves them running rather + # than kill them mid-shutdown, so stop here if it could not stop them. + ERROR_HINT="Rerun the installer to finish shutting down; nothing needs to be undone." + combinectl stop || error "Could not stop The Combine." + ERROR_HINT="" # Disable The Combine services from starting at boot time sudo systemctl disable create_ap sudo systemctl disable k3s diff --git a/docs/deploy/README.md b/docs/deploy/README.md index ec9de372ff..bc5c621a83 100644 --- a/docs/deploy/README.md +++ b/docs/deploy/README.md @@ -423,14 +423,38 @@ Notes: - When the `./setup_combine.py` script is used to install _The Combine_ on a NUC, it will install the fonts required for Arabic, English, French, Portuguese, and Spanish. If additional fonts will be required, call the `setup_combine.py` commands with the `--langs` option. Use the `--help` option to see the argument syntax. -- The database pod has a `postStart` lifecycle hook that runs `update-semantic-domains.sh` automatically on every pod - start, but only imports data if the `SemanticDomainTree` or `SemanticDomains` collections are empty. If the Semantic - Domain data are updated, for example, adding a new language, then the script needs to be rerun manually: +- The database pod has a `postStart` lifecycle hook that initializes the `rs0` replica set on every pod start and, if + the import has not already been recorded as complete, runs `update-semantic-domains.sh`. That script records its own + completion in `CombineDatabase.SemanticDomainImportStatus`, and only on success; a count of the imported collections + is not used, because an import that is interrupted part way through leaves them non-empty but incomplete. If the + import is interrupted, the next pod start redoes it. If the import fails outright, the hook fails, and the kubelet + restarts the container to retry; _The Combine_ cannot be used without the semantic domains, so this is preferred over + a database that looks healthy without them. Look in the `postStart` log, below, to see why an import is failing. + + The hook then writes `/tmp/replica-set-ready`, which is what the pod's readiness probe checks. Until then the pod is + kept out of the `database` Service endpoints, since the replica set advertises the pod's IP and the backend connects + with `?replicaSet=rs0`. Note that the kubelet does not probe a container until its `postStart` hook returns, so the + pod cannot become ready during the import no matter when the marker is written. + + The completion record is in the database's persistent volume, so it outlives the pod that wrote it: once an import is + recorded, no later pod start imports again. An installation that imported before the record existed does not have one, + so its first pod start on this chart imports once more, and the pod stays out of the `database` Service for the few + minutes that takes. Whenever the Semantic Domain data change, the import has to be rerun manually. That is true both + of data added by hand, for example a new language, and of a release that ships an updated `tree.json` or `nodes.json`, + including one installed with `combinectl update`; neither is picked up on its own. A manual run refreshes the + completion record, so it is not redone on the next pod start: ```console kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh ``` + The `postStart` hook appends its output to `/data/db/postStart.log`, which is on the database's persistent volume and + so survives restarts. Only the last 200 lines are kept, so the oldest entry in it may begin part way through: + + ```console + kubectl -n thecombine exec deployment/database -- cat /data/db/postStart.log + ``` + ## Maintenance ### Maintenance Scripts for Kubernetes diff --git a/installer/README.md b/installer/README.md index 677d0b8410..ca6189d6c9 100644 --- a/installer/README.md +++ b/installer/README.md @@ -162,9 +162,9 @@ To run `combine-installer.run` with options, the option list must be started wit | clean | Remove the previously saved environment (AWS Access Key, admin user info) and any previously saved timeout before performing the installation. | | restart | Run the installation from the beginning; do not resume a previous installation. | | server | Install _The Combine_ in a server environment so that _The Combine_ is always running by default. | -| timeout TIMEOUT | Use a different timeout when installing. (Default: 5 minutes.) With slow internet, it is helpful to extend the timeout. See for timeout formats. The value is used for the rest of the installation, including after a restart, so it does not need to be entered again. | +| timeout TIMEOUT | Use a different timeout when installing. (Default: 5 minutes.) With slow internet, it is helpful to extend the timeout. See for timeout formats. The value is used for the rest of the installation, including after a restart, so it does not need to be entered again. On a first installation the timeout also has to cover the semantic domain import, which takes several minutes; if it runs out, rerun the installer and it continues from where it stopped. | | uninstall | Remove software installed by this script. | -| update | Update _The Combine_ to the version number provided. This skips installing support software that was installed previously. | +| update | Update _The Combine_ to the version number provided. This skips installing support software that was installed previously, which includes the `combinectl` tool, so run the installation without this option to pick up changes to it. | | version-number | Specify a version to install instead of the current version. A version number will have the form `vn.n.n` where `n` represents an integer value, for example, `v1.20.0`. | ### Examples diff --git a/nginx/init/05-wait-for-backend-dns.sh b/nginx/init/05-wait-for-backend-dns.sh new file mode 100755 index 0000000000..30efd69acc --- /dev/null +++ b/nginx/init/05-wait-for-backend-dns.sh @@ -0,0 +1,34 @@ +#! /bin/bash + +############################################################### +# nginx resolves the hostnames in its proxy_pass directives when +# it loads its configuration and exits if any of them cannot be +# resolved: +# +# [emerg] host not found in upstream "backend" +# +# On a cold start the frontend container can be running before +# the cluster DNS is able to answer for the backend service, so +# wait for the name before letting nginx start. +############################################################### + +BACKEND_HOST=backend +MAX_WAIT_SECONDS=120 + +if getent hosts "${BACKEND_HOST}" > /dev/null 2>&1 ; then + exit 0 +fi + +echo "Waiting up to ${MAX_WAIT_SECONDS}s for '${BACKEND_HOST}' to resolve" +WAITED=0 +until getent hosts "${BACKEND_HOST}" > /dev/null 2>&1 ; do + if [ "${WAITED}" -ge "${MAX_WAIT_SECONDS}" ] ; then + # Start nginx anyway so that it reports the problem itself, rather than + # leaving a container that is running but never serving. + echo "'${BACKEND_HOST}' did not resolve after ${MAX_WAIT_SECONDS}s" + exit 0 + fi + sleep 2 + WAITED=$((WAITED + 2)) +done +echo "'${BACKEND_HOST}' resolved after ${WAITED}s" diff --git a/nginx/templates/default.conf.template b/nginx/templates/default.conf.template index 888376c484..a4c21b97d4 100644 --- a/nginx/templates/default.conf.template +++ b/nginx/templates/default.conf.template @@ -44,6 +44,9 @@ server { text/x-cross-domain-policy; # text/html is always compressed by gzip module + # nginx exits if a proxy_pass host cannot be resolved when it loads this + # configuration, so every cluster-internal host used below has to be waited + # for in nginx/init/05-wait-for-backend-dns.sh. location /v1 { proxy_pass http://backend:5000; proxy_http_version 1.1;