Fix/laptop install first boot

database/init/update-semantic-domains.sh

@@ -1,4 +1,15 @@
 #! /usr/bin/bash
+# A partial import leaves the collections non-empty but incomplete, so record
+# completion here and only on success. Doing it here rather than in the caller
+# means a manual run also counts, and stops the next container start from
+# redoing the whole import.
+#
+# Stop at the first failure and report it: The Combine cannot be used without the
+# semantic domains, so the database's postStart hook restarts the container on a
+# non-zero exit rather than leave a database that looks healthy without them.
+set -eo pipefail
 
 mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang
 mongoimport -d CombineDatabase -c SemanticDomains /data/semantic-domains/nodes.json --mode=merge --upsertFields=id,guid,lang
+
+mongosh --quiet --host 127.0.0.1 --eval "db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.replaceOne({ _id: 'semantic-domains' }, { _id: 'semantic-domains', completed: true }, { upsert: true });"

deploy/ansible/roles/support_tools/files/combinectl.sh

@@ -31,9 +31,8 @@ usage () {
 .EOM
 }
 
-# Get the name of the first wifi interface.  In general,
-# this script assumes that there is a single WiFi interface
-# installed.
+# Get the name of the first wifi interface. In general, this script assumes
+# that there is a single WiFi interface installed.
 get-wifi-if () {
   IFS=$'\n' WIFI_DEVICES=( $(nmcli d | grep "^wl") )
   if [[ ${#WIFI_DEVICES[@]} -gt 0 ]] ; then
@@ -73,7 +72,133 @@ combine-cert () {
   echo $CERT_DATA | base64 -d | openssl x509 -enddate -noout| sed -e "s/^notAfter=/Web certificate expires at /"
 }
 
-# Start The Combine services
+# Report whether the Kubernetes API is serving requests. The k3s unit becomes
+# active before the API is up, so an active unit alone is not enough.
+cluster-ready () {
+  kubectl get --raw='/readyz' --request-timeout=10s > /dev/null 2>&1
+}
+
+# Wait up to two minutes for the Kubernetes API to serve requests.
+wait-for-cluster () {
+  ATTEMPTS=0
+  until cluster-ready ; do
+    ATTEMPTS=$((ATTEMPTS + 1))
+    if [[ ${ATTEMPTS} -ge 60 ]] ; then
+      return 1
+    fi
+    sleep 2
+  done
+  return 0
+}
+
+# Print "name requested available" for every deployment in the namespace.
+# availableReplicas is absent, rather than 0, when none are available.
+combine-deployments () {
+  kubectl -n thecombine get deployments 2> /dev/null \
+    -o 'jsonpath={range .items[*]}{.metadata.name} {.spec.replicas} {.status.availableReplicas}{"\n"}{end}'
+}
+
+# Restore the deployment replica counts saved by stop-combine-deployments. The
+# counts live in a local file, so fall back to one replica each when it is
+# missing: k3s can be started without combinectl, which would otherwise leave
+# the deployments scaled to zero with nothing to bring them back.
+#
+# Returns non-zero when a deployment that should be running is not, so that a
+# failed start is not reported as a successful one.
+start-combine-deployments () {
+  if ! wait-for-cluster ; then
+    echo "The cluster is not responding; run \"combinectl start\" again." >&2
+    return 1
+  fi
+  DEPLOY_STATUS=$(combine-deployments)
+  if [[ -z ${DEPLOY_STATUS} ]] ; then
+    # Nothing is installed, so there is nothing to start; combine-status is
+    # where an empty namespace is reported.
+    return 0
+  fi
+  if [ -f "${CACHED_REPLICAS}" ] ; then
+    CACHE_FILE="${CACHED_REPLICAS}"
+  else
+    CACHE_FILE=/dev/null
+  fi
+  # List every deployment that is scaled down, with its saved count. The saved
+  # counts are reconciled with the cluster rather than replayed as they are: a
+  # cached deployment that no longer exists, for example one renamed by a chart
+  # update, would otherwise fail to scale on every start, and its failure would
+  # keep the stale file, and any deployment the file omits, forever.
+  REPLICA_LIST=$(awk '
+    NR == FNR { if ($2 == 0) { stopped[$1] = 1 } ; next }
+    $1 in stopped && $2 > 0 { print $1, $2 ; delete stopped[$1] }
+    END { for (name in stopped) { print name, 1 } }
+    ' <(printf '%s\n' "${DEPLOY_STATUS}") "${CACHE_FILE}")
+  if [[ -z ${REPLICA_LIST} ]] ; then
+    # Nothing is scaled down, so any saved counts no longer apply.
+    rm -f "${CACHED_REPLICAS}"
+    return 0
+  fi
+  if [[ ${CACHE_FILE} == /dev/null ]] ; then
+    echo "No saved replica counts; starting one replica of each deployment."
+  fi
+  echo "Starting The Combine deployments."
+  RESTORE_FAILED=0
+  while read -r DEPLOYMENT REPLICAS ; do
+    if [[ -z ${DEPLOYMENT} || -z ${REPLICAS} ]] ; then
+      continue
+    fi
+    if ! kubectl -n thecombine scale "deployment/${DEPLOYMENT}" --replicas="${REPLICAS}" > /dev/null ; then
+      echo "Could not start deployment/${DEPLOYMENT}." >&2
+      RESTORE_FAILED=1
+    fi
+  done <<< "${REPLICA_LIST}"
+  # Keep the counts for the next attempt if any deployment was not restored.
+  if [[ ${RESTORE_FAILED} -eq 0 ]] ; then
+    rm -f "${CACHED_REPLICAS}"
+  fi
+  return ${RESTORE_FAILED}
+}
+
+# Scale The Combine deployments to zero and wait for their pods to exit. The
+# k3s service is patched to KillMode=mixed, so stopping it SIGKILLs whatever is
+# still running, which can be the database part way through its startup setup.
+#
+# Returns non-zero if the cluster could not be reached, so that the caller leaves
+# k3s running. A pod still terminating after two minutes only warns, so that one
+# stuck pod cannot leave The Combine with no way to stop.
+stop-combine-deployments () {
+  # An unreachable Kubernetes API looks exactly like an empty namespace, so wait
+  # for it: a stop issued while the cluster is still coming up must not be read
+  # as "nothing is running here."
+  if ! wait-for-cluster ; then
+    echo "The cluster is not responding, so The Combine was not stopped." >&2
+    echo "Wait a minute, then run \"combinectl stop\" again." >&2
+    return 1
+  fi
+  DEPLOY_STATUS=$(combine-deployments)
+  if [[ -z ${DEPLOY_STATUS} ]] ; then
+    return 0
+  fi
+  # Only record the deployments that are running, so that stopping The Combine
+  # when it's already stopped doesn't lose the counts.
+  REPLICA_LIST=$(awk '$2 > 0 { print $1, $2 }' <<< "${DEPLOY_STATUS}")
+  if [[ -n ${REPLICA_LIST} ]] ; then
+    echo "${REPLICA_LIST}" > "${CACHED_REPLICAS}"
+  fi
+  echo "Stopping The Combine deployments."
+  kubectl -n thecombine scale deployment --all --replicas=0 > /dev/null
+  # A selector-based "kubectl wait" fails immediately when nothing matches it,
+  # so only wait when there are pods left to wait for.
+  if [[ -n $(kubectl -n thecombine get pods -l combine-component --no-headers 2> /dev/null) ]] ; then
+    if ! kubectl -n thecombine wait --for=delete pod -l combine-component \
+        --timeout=2m > /dev/null 2>&1 ; then
+      echo "The Combine did not stop within 2 minutes; stopping anyway." >&2
+    fi
+  fi
+  return 0
+}
+
+# Start The Combine services. The status of the last command is the status of
+# the function, so this returns non-zero if the deployments were not started,
+# which matches combine-stop.
 combine-start () {
   echo "Starting The Combine."
   if ! systemctl is-active --quiet create_ap ; then
@@ -84,42 +209,108 @@ combine-start () {
   if ! systemctl is-active --quiet k3s ; then
     sudo systemctl start k3s
   fi
+  start-combine-deployments
 }
 
-# Stop The Combine services and restore the WiFI
-# connection if needed.
+# Stop The Combine services and restore the WiFi connection if needed. Returns
+# non-zero if The Combine is still running.
 combine-stop () {
   echo "Stopping The Combine."
   if systemctl is-active --quiet k3s ; then
+    # Stopping k3s SIGKILLs the containers, so only do it once the deployments
+    # have shut down; leave everything running otherwise.
+    if ! stop-combine-deployments ; then
+      return 1
+    fi
     sudo systemctl stop k3s
   fi
   if systemctl is-active --quiet create_ap ; then
     sudo systemctl stop create_ap
     restore-wifi-connection
     sudo systemctl restart systemd-resolved
   fi
+  return 0
 }
 
-# Print the status of The Combine services.  If the combine is
-# "up" then also print that status of the deployments in
-# "thecombine" namespace.
+# Print the status of The Combine services. When the cluster is up, also
+# distinguish between The Combine being uninstalled, incompletely installed,
+# scaled down, partly scaled down, still starting, and fully running, then print
+# the status of the deployments in the "thecombine" namespace.
+#
+# Always exits 0; install-combine.sh calls this under "set -e".
 combine-status () {
   if systemctl is-active --quiet create_ap ; then
     echo "WiFi hotspot is Running."
   else
     echo "WiFi hotspot is Stopped."
   fi
-  if systemctl is-active --quiet k3s ; then
-    echo "The Combine  is Running."
-    kubectl -n thecombine get deployments
-  else
+
+  if ! systemctl is-active --quiet k3s ; then
     echo "The Combine is Stopped."
+    return 0
+  fi
+
+  if ! cluster-ready ; then
+    echo "The Combine is Starting; the Kubernetes cluster is not ready yet."
+    echo "Wait a minute, then run \"combinectl status\" again."
+    return 0
+  fi
+
+  DEPLOY_STATUS=$(combine-deployments)
+  if [[ -z ${DEPLOY_STATUS} ]] ; then
+    echo "The Combine is Not Installed; the Kubernetes cluster is running, but"
+    echo "the \"thecombine\" namespace has no deployments."
+    echo "Download and run the install package to install The Combine."
+    return 0
+  fi
+
+  MISSING=()
+  for DEPLOYMENT in "${COMBINE_DEPLOYMENTS[@]}" ; do
+    if ! grep -q "^${DEPLOYMENT} " <<< "${DEPLOY_STATUS}" ; then
+      MISSING+=( "${DEPLOYMENT}" )
+    fi
+  done
+
+  # Total requested replicas, to tell a scaled down Combine from a running one,
+  # the deployments that ask for no replicas at all, and the ones that do not
+  # have all of the replicas they asked for. A deployment scaled to zero has
+  # every replica it asked for, so it has to be counted separately from those.
+  REQUESTED=0
+  STOPPED=()
+  PENDING=()
+  while read -r NAME WANT HAVE ; do
+    if [[ -z ${NAME} ]] ; then
+      continue
+    fi
+    REQUESTED=$(( REQUESTED + ${WANT:-0} ))
+    if [[ ${WANT:-0} -eq 0 ]] ; then
+      STOPPED+=( "${NAME}" )
+    elif [[ ${HAVE:-0} -lt ${WANT:-0} ]] ; then
+      PENDING+=( "${NAME}" )
+    fi
+  done <<< "${DEPLOY_STATUS}"
+
+  if [[ ${#MISSING[@]} -gt 0 ]] ; then
+    echo "The Combine is Incomplete; missing deployment(s): ${MISSING[*]}."
+    echo "Download and run the install package to repair the installation."
+  elif [[ ${REQUESTED} -eq 0 ]] ; then
+    echo "The Combine is Stopped; the cluster is up but its services are"
+    echo "scaled down. Run \"combinectl start\" to start them."
+  elif [[ ${#STOPPED[@]} -gt 0 ]] ; then
+    echo "The Combine is Partly Stopped; scaled down deployment(s): ${STOPPED[*]}."
+    echo "Run \"combinectl start\" to start them."
+  elif [[ ${#PENDING[@]} -gt 0 ]] ; then
+    echo "The Combine is Starting; waiting for: ${PENDING[*]}."
+  else
+    echo "The Combine is Running."
   fi
+  kubectl -n thecombine get deployments
+  return 0
 }
 
-# Update the image used in each of the deployments in The Combine.  This
-# is akin to our current update process for Production and QA servers.  It
-# does *not* update any configuration files or secrets.
+# Update the image used in each of the deployments in The Combine. This is akin
+# to our current update process for Production and QA servers. It does *not*
+# update any configuration files or secrets.
 combine-update () {
   echo "Updating The Combine to $1"
   IMAGE_TAG=$1
@@ -155,11 +346,15 @@ combine-wifi-set-password () {
 }
 
 # Main script entrypoint
+# The deployments that make up The Combine, used to detect an installation that
+# is missing components. Matches the list in install-combine.sh.
+COMBINE_DEPLOYMENTS=(backend database frontend maintenance)
 WIFI_IF=$(get-wifi-if)
 WIFI_CONFIG=/etc/create_ap/create_ap.conf
 export KUBECONFIG=${HOME}/.kube/config
 COMBINE_CONFIG=${HOME}/.config/combine
 CACHED_WIFI_CONN=${COMBINE_CONFIG}/wifi-connection.txt
+CACHED_REPLICAS=${COMBINE_CONFIG}/deployment-replicas.txt
 
 # Make sure config directory exists
 mkdir -p "${COMBINE_CONFIG}"

deploy/helm/thecombine/charts/database/templates/database.yaml

@@ -56,8 +56,17 @@ spec:
                   - /bin/sh
                   - -c
                   - |
-                    exec > /data/db/postStart.log 2>&1
+                    log=/data/db/postStart.log
+                    # One block is appended per container start. Keeping the last
+                    # 200 lines keeps the recent starts and drops the rest, but it
+                    # does not respect block boundaries, so the oldest block that
+                    # survives can begin part way through.
+                    if [ -f "${log}" ]; then
+                      tail -n 200 "${log}" > "${log}.trim" && mv "${log}.trim" "${log}"
+                    fi
+                    exec >> "${log}" 2>&1
                     set -e
+                    echo "[postStart] $(date -Is) starting"
                     echo "[postStart] Waiting for mongod to accept connections"
                     attempts=0
                     until mongosh --quiet --host 127.0.0.1 --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; do
@@ -70,10 +79,27 @@ spec:
                     done
                     echo "[postStart] Ensuring replica set host"
                     mongosh --quiet --host 127.0.0.1 /opt/thecombine/00-replica-set.js || exit $?
-                    needs_semantic_import="$(mongosh --quiet --host 127.0.0.1 --eval "const combineDb = db.getSiblingDB('CombineDatabase'); const treeCount = combineDb.SemanticDomainTree.countDocuments({}); const domainCount = combineDb.SemanticDomains.countDocuments({}); print(treeCount === 0 || domainCount === 0 ? 'yes' : 'no');")"
-                    if [ "${needs_semantic_import}" = "yes" ]; then
-                      /bin/bash /opt/thecombine/update-semantic-domains.sh
+                    # Only a finished import is recorded, so an interrupted one is
+                    # redone. The record is read by exit status; anything short of a
+                    # completed import, including a query that fails, imports again,
+                    # which is a merge and so safe to repeat. Only stdout is discarded,
+                    # to leave any error mongosh reports in this log.
+                    import_done="quit(db.getSiblingDB('CombineDatabase').SemanticDomainImportStatus.countDocuments({ _id: 'semantic-domains', completed: true }) === 1 ? 0 : 1)"
+                    if ! mongosh --quiet --host 127.0.0.1 --eval "${import_done}" > /dev/null; then
+                      echo "[postStart] Importing semantic domains"
+                      # The Combine cannot be used without the semantic domains, so a
+                      # failed import fails the hook, which restarts the container to
+                      # retry, rather than leaving a database that looks healthy.
+                      if ! /bin/bash /opt/thecombine/update-semantic-domains.sh; then
+                        echo "[postStart] Semantic domain import failed; restarting the container"
+                        exit 1
+                      fi
                     fi
+                    # The kubelet does not probe a container until its postStart hook
+                    # returns, so this marker is written last: it cannot make the pod
+                    # ready any sooner, and everything above it has to succeed first.
+                    touch /tmp/replica-set-ready
+                    echo "[postStart] $(date -Is) done"
           env:
             - name: POD_IP
               valueFrom:
@@ -83,6 +109,21 @@ spec:
               value: "$(POD_IP):27017"
           ports:
             - containerPort: 27017
+          readinessProbe:
+            # /tmp/replica-set-ready is written at the end of the postStart hook,
+            # once mongod is a writable primary advertising this pod's IP and the
+            # semantic domains are in place. The pod IP is part of the replica set
+            # config and changes on every restart, so without this the Service can
+            # route the backend to a mongod it cannot use. A hook that fails, and
+            # so restarts the container, never writes the marker at all.
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - test -f /tmp/replica-set-ready
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 5
           resources:
             requests:
               cpu:  25m

deploy/helm/thecombine/charts/frontend/templates/deployment-frontend.yaml

@@ -67,6 +67,13 @@ spec:
           ports:
             - containerPort: 80
             - containerPort: 443
+          readinessProbe:
+            # The entrypoint waits for cluster DNS to resolve the backend before
+            # starting nginx, so a running container is not yet a serving one.
+            tcpSocket:
+              port: 80
+            initialDelaySeconds: 5
+            periodSeconds: 5
           resources:
             requests:
               cpu:  1m