feat(openbao): support KMS auto-unseal in the self-managed chart

deploy/helm/openbao/cleanup.sh

@@ -39,12 +39,16 @@ cleanup_cluster() {
     fi
     log_success "Cleaned up OpenBao PVCs in namespace: $namespace"
 
-    # Delete the unseal key and root token secret
+    # Delete the unseal key, recovery keys, and root token secret
     log_info "Deleting secrets in namespace: $namespace"
     if ! kubectl delete secret $statefulset-unseal -n $namespace --ignore-not-found=true > /dev/null 2>&1; then
         log_error "Failed to delete unseal secret (exit code: $?): $?"
         exit 1
     fi
+    if ! kubectl delete secret $statefulset-recovery-keys -n $namespace --ignore-not-found=true > /dev/null 2>&1; then
+        log_error "Failed to delete recovery-keys secret (exit code: $?): $?"
+        exit 1
+    fi
     if ! kubectl delete secret $statefulset-root-token -n $namespace --ignore-not-found=true > /dev/null 2>&1; then
         log_error "Failed to delete root token secret (exit code: $?): $?"
         exit 1

deploy/helm/openbao/helm/scripts/deploy.sh

@@ -59,6 +59,67 @@ get_root_token() {
     kubectl get secret ${statefulset}-root-token -n ${namespace} -o jsonpath='{.data.root_token}' | base64 -d
 }
 
+# Resolve the seal mode to use for init and unseal, into the global
+# RESOLVED_SEAL_MODE ("auto" or "shamir"). Returns non-zero (fail closed) on an
+# unreadable status or a mismatch, rather than guessing a path that could
+# discard the only copy of a key.
+#
+# AUTO_UNSEAL is the chart's declared intent (server.autoUnseal.enabled), which
+# also gates whether the unseal Secret exists. `bao status` reports the server's
+# actual seal: recovery_seal=true for an auto-unseal (KMS or HSM) seal such as
+# awskms, azurekeyvault, gcpckms, transit, or pkcs11; false for Shamir. These
+# must agree: a flag set without the matching seal stanza (or the reverse) would
+# otherwise take a path whose Secret does not exist.
+#
+# `bao status` exits 0 when unsealed and 2 when sealed; both return valid JSON.
+# Any other exit (1) is a real error. jq -r is used, not jq -e, because a valid
+# `false` makes jq -e exit non-zero.
+RESOLVED_SEAL_MODE=""
+resolve_seal_mode() {
+    local namespace=$1
+    local statefulset=$2
+    local declared="${AUTO_UNSEAL:-false}"
+
+    local out rc
+    out=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \
+        bao status -format=json 2>/dev/null)
+    rc=$?
+    if [ "${rc}" != "0" ] && [ "${rc}" != "2" ]; then
+        log_error "Could not read seal status from ${statefulset}-0 (bao status exit ${rc})"
+        return 1
+    fi
+
+    # Parse recovery_seal, failing closed on unparseable or non-boolean status.
+    # A missing field stays on the Shamir path (matches the pre-existing default
+    # for an uninitialized server); only an explicit boolean true selects auto.
+    # jq -r (not -e) is used so a valid false does not look like a jq failure.
+    local recovery_seal
+    recovery_seal=$(printf '%s' "${out}" | jq -rs '
+        if (length != 1) or ((.[0] | type) != "object") then error("invalid status")
+        elif (.[0] | has("recovery_seal") | not) then false
+        elif (.[0].recovery_seal | type) != "boolean" then error("recovery_seal not boolean")
+        else .[0].recovery_seal end' 2>/dev/null) || {
+        log_error "Could not parse seal status from ${statefulset}-0"
+        return 1
+    }
+    local server_mode="shamir"
+    if [ "${recovery_seal}" = "true" ]; then
+        server_mode="auto"
+    fi
+
+    if [ "${declared}" = "true" ] && [ "${server_mode}" != "auto" ]; then
+        log_error "server.autoUnseal.enabled is set but the server reports no auto-unseal seal. Add the seal stanza to server.ha.raft.config (see values-autounseal.yaml.example)."
+        return 1
+    fi
+    if [ "${declared}" != "true" ] && [ "${server_mode}" = "auto" ]; then
+        log_error "The server reports an auto-unseal seal but server.autoUnseal.enabled is not set. Enable it so the unseal Secret and init path match the seal."
+        return 1
+    fi
+
+    RESOLVED_SEAL_MODE="${server_mode}"
+    return 0
+}
+
 # Runtime version-skew check: verify that the auto-unseal-sidecar container's
 # image tag matches the openbao server container's image tag in the live
 # StatefulSet spec. This complements the template-time guard in
@@ -188,29 +249,84 @@ initialize_cluster() {
     log_info "All OpenBao pods are ready"
 
     log_info "Initializing OpenBao cluster"
-    local init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \
-        bao operator init \
-        -key-shares=1 \
-        -key-threshold=1 \
-        -format=json)
-
-    # Extract keys
-    local unseal_key=$(echo ${init_output} | jq -r '.unseal_keys_b64[0]')
-    local root_token=$(echo ${init_output} | jq -r '.root_token')
-
-    # Check if unseal key is empty
-    if [ -z "${unseal_key}" ]; then
-        log_error "Failed to get unseal key from initialization output"
+    if ! resolve_seal_mode "${namespace}" "${statefulset}"; then
         return 1
     fi
-
-    # Update the secret with the new unseal key
-    kubectl patch secret ${statefulset}-unseal \
-        --patch "data:
+    local init_output
+    local root_token
+    if [ "${RESOLVED_SEAL_MODE}" = "auto" ]; then
+        # Auto-unseal seal: initialize with recovery keys, not an unseal key.
+        # Recovery keys regenerate the root token and rekey; they never unseal
+        # (the seal does that), so no unseal key or unseal Secret is needed.
+        local recovery_shares="${RECOVERY_SHARES:-5}"
+        local recovery_threshold="${RECOVERY_THRESHOLD:-3}"
+        log_info "Auto-unseal seal detected; initializing with ${recovery_shares} recovery shares (threshold ${recovery_threshold})"
+        if ! init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \
+            bao operator init \
+            -recovery-shares=${recovery_shares} \
+            -recovery-threshold=${recovery_threshold} \
+            -format=json); then
+            log_error "bao operator init failed. If the server is now initialized its keys were not captured; tear down with cleanup.sh and reinitialize."
+            return 1
+        fi
+        root_token=$(echo ${init_output} | jq -r '.root_token')
+        if [ -z "${root_token}" ] || [ "${root_token}" = "null" ]; then
+            log_error "Failed to get root token from initialization output"
+            return 1
+        fi
+        # Persist the recovery keys before anything else can discard them: once
+        # the server is initialized, `operator init` cannot reproduce them, so a
+        # failed write here would lose the only copy and make break-glass
+        # recovery impossible. Retry, and fail the init if it cannot be stored
+        # rather than reporting success without keys.
+        local recovery_keys
+        recovery_keys=$(echo ${init_output} | jq -c '.recovery_keys_b64')
+        if [ -z "${recovery_keys}" ] || [ "${recovery_keys}" = "null" ]; then
+            log_error "Initialization did not return recovery keys (.recovery_keys_b64 is null). Aborting."
+            return 1
+        fi
+        local persisted=false
+        for attempt in 1 2 3; do
+            if kubectl create secret generic ${statefulset}-recovery-keys \
+                -n ${namespace} \
+                --from-literal=recovery_keys_b64="${recovery_keys}" \
+                --dry-run=client -o yaml | kubectl apply -f -; then
+                persisted=true
+                break
+            fi
+            log_warn "Could not persist recovery keys (attempt ${attempt}/3); retrying..."
+            sleep 3
+        done
+        if [ "${persisted}" != "true" ]; then
+            log_error "Failed to store recovery keys in Secret '${statefulset}-recovery-keys'. The server is initialized but the keys are not saved, so break-glass recovery is impossible. Tear down with cleanup.sh and reinitialize."
+            return 1
+        fi
+        log_warn "Recovery keys stored in Secret '${statefulset}-recovery-keys'. Export them to a break-glass store and delete this Secret."
+    else
+        # Shamir seal: a single unseal key, stored for the auto-unseal sidecar.
+        if ! init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \
+            bao operator init \
+            -key-shares=1 \
+            -key-threshold=1 \
+            -format=json); then
+            log_error "bao operator init failed. If the server is now initialized its keys were not captured; tear down with cleanup.sh and reinitialize."
+            return 1
+        fi
+        local unseal_key=$(echo ${init_output} | jq -r '.unseal_keys_b64[0]')
+        root_token=$(echo ${init_output} | jq -r '.root_token')
+        if [ -z "${unseal_key}" ] || [ "${unseal_key}" = "null" ]; then
+            log_error "Failed to get unseal key from initialization output"
+            return 1
+        fi
+        if ! kubectl patch secret ${statefulset}-unseal \
+            --patch "data:
     unseal_key: $(echo -n "${unseal_key}" | base64)" \
-        -n ${namespace}
-
-    log_info "Updated Kubernetes secret '${statefulset}-unseal' with unseal key"
+            -n ${namespace}; then
+            log_error "Failed to store unseal key in Secret '${statefulset}-unseal'. The server is initialized but the unseal key is not saved; tear down with cleanup.sh and reinitialize."
+            return 1
+        fi
+        log_info "Updated Kubernetes secret '${statefulset}-unseal' with unseal key"
+    fi
 
     # Store root token in a new secret
     log_info "Creating secret '${statefulset}-root-token' with root token..."
@@ -229,10 +345,34 @@ get_unseal_key() {
 unseal_cluster() {
     local namespace=$1
     local statefulset=$2
-    local unseal_key=$(get_unseal_key "${namespace}" "${statefulset}")
 
     log_section "Unsealing OpenBao cluster"
 
+    if ! resolve_seal_mode "${namespace}" "${statefulset}"; then
+        return 1
+    fi
+    if [ "${RESOLVED_SEAL_MODE}" = "auto" ]; then
+        # The auto-unseal seal unseals each node on start, and retry_join in the
+        # raft config joins the peers. No manual unseal or raft join is needed;
+        # wait for every pod to report unsealed.
+        log_info "Auto-unseal seal detected; waiting for all pods to unseal"
+        local end=$((SECONDS + 120))
+        for i in {0..2}; do
+            while [ "$(kubectl exec ${statefulset}-${i} -c openbao -n ${namespace} -- bao status -format=json 2>/dev/null | jq -r '.sealed' 2>/dev/null)" != "false" ]; do
+                if [ $SECONDS -gt $end ]; then
+                    log_error "Timeout waiting for pod ${statefulset}-${i} to auto-unseal"
+                    return 1
+                fi
+                log_info "Waiting for ${statefulset}-${i} to auto-unseal..."
+                sleep 5
+            done
+        done
+        log_info "All pods auto-unsealed"
+        return 0
+    fi
+
+    local unseal_key=$(get_unseal_key "${namespace}" "${statefulset}")
+
     # First unseal the primary node (pod 0)
     log_info "Unsealing primary pod ${statefulset}-0"
     if ! kubectl exec ${statefulset}-0 -n ${namespace} -- \

deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml

@@ -52,6 +52,15 @@ spec:
           args:
             - -c
             - /scripts/deploy.sh {{ include "nvcf-openbao.namespace" . }} {{ $serverFullname }} helm
+          env:
+            - name: AUTO_UNSEAL
+              value: {{ .Values.openbao.server.autoUnseal.enabled | quote }}
+            {{- if .Values.openbao.server.autoUnseal.enabled }}
+            - name: RECOVERY_SHARES
+              value: {{ .Values.openbao.server.autoUnseal.recovery.shares | quote }}
+            - name: RECOVERY_THRESHOLD
+              value: {{ .Values.openbao.server.autoUnseal.recovery.threshold | quote }}
+            {{- end }}
           volumeMounts:
             - name: init-script
               readOnly: true

deploy/helm/openbao/helm/templates/hook-pre-01-unseal-secret.yaml

@@ -14,6 +14,9 @@
 # limitations under the License.
 
 {{- $serverFullname := include "nvcf-openbao.serverFullname" . }}
+{{- /* KMS auto-unseal does not use a Shamir unseal key, so skip the empty
+       unseal Secret when server.autoUnseal.enabled is set. */ -}}
+{{- if not .Values.openbao.server.autoUnseal.enabled }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -26,3 +29,4 @@ metadata:
 type: Opaque
 data:
   unseal_key: ""
+{{- end }}

deploy/helm/openbao/helm/values-autounseal.yaml.example

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Example values overlay for OpenBao auto-unseal. Apply on top of values.yaml.
+#
+# This turns on server.autoUnseal, adds a seal stanza so each node unseals
+# through the seal on start, and drops the Shamir auto-unseal sidecar and its
+# unseal Secret volume. The init hook then initializes with recovery keys.
+#
+# The seal block below uses AWS KMS as a concrete example. For Azure Key Vault,
+# GCP KMS, or HashiCorp/OpenBao Transit, replace just the seal stanza with that
+# provider's block (see the OpenBao seal documentation); the rest of this overlay
+# is identical.
+#
+# A PKCS#11 HSM is not a drop-in stanza swap. The seal "pkcs11" block points at
+# the HSM vendor's PKCS#11 library (lib, token_label, key_label), which must be
+# present in the server container. PKCS#11 support itself comes from either an
+# HSM-enabled OpenBao build with PKCS#11 compiled in via cgo, or the external
+# PKCS#11 KMS provider plugin (openbao-plugins). Make sure the OpenBao image you
+# run provides one of these; the seal stanza alone does not. See the OpenBao
+# pkcs11 seal docs.
+#
+# The server pod needs the provider's unwrap permission on the key, granted out
+# of band. For AWS KMS that is kms:Encrypt, kms:Decrypt and kms:DescribeKey, for
+# example through an EKS Pod Identity or IRSA role. Automatic key rotation is
+# safe where the provider keeps the key id and can still decrypt the previously
+# wrapped root key.
+#
+# Required post-install step, before treating the cluster as production. The init
+# hook stores all recovery shares in one Secret, <statefulset>-recovery-keys.
+# While it exists, any reader of that Secret holds the full recovery quorum, so
+# the shares/threshold below give no protection. Finish the install by exporting
+# the shares, splitting them among separate custodians offline, and deleting the
+# Secret:
+#
+#   kubectl get secret <statefulset>-recovery-keys -n <namespace> \
+#     -o jsonpath='{.data.recovery_keys_b64}' | base64 -d   # then split offline
+#   kubectl delete secret <statefulset>-recovery-keys -n <namespace>
+#
+# Only after the delete does the threshold protect the quorum. deploy.sh logs the
+# same reminder at the end of init.
+
+openbao:
+  server:
+    autoUnseal:
+      enabled: true
+      recovery:
+        shares: 5
+        threshold: 3
+
+    # Override the raft config to add the seal stanza. Keep the rest in sync
+    # with values.yaml; only the seal block is new here.
+    ha:
+      raft:
+        config: |
+          ui = true
+          storage "raft" {
+            path = "/openbao/data/"
+            retry_join {
+              leader_api_addr = "http://openbao-server-0.openbao-server-internal:8200"
+            }
+            retry_join {
+              leader_api_addr = "http://openbao-server-1.openbao-server-internal:8200"
+            }
+            retry_join {
+              leader_api_addr = "http://openbao-server-2.openbao-server-internal:8200"
+            }
+          }
+          listener "tcp" {
+              tls_disable = 1
+              address = "[::]:8200"
+              cluster_address = "[::]:8201"
+
+              custom_response_headers {
+                "default" = {
+                  "X-Custom-Header" = ["HOSTNAME"],
+                },
+              }
+          }
+          # Example: AWS KMS. Swap this block for azurekeyvault, gcpckms,
+          # transit, or pkcs11 to use a different provider.
+          seal "awskms" {
+            region     = "<region>"
+            kms_key_id = "<kms-key-id>"
+          }
+          plugin_directory = "/openbao/plugins/"
+          service_registration "kubernetes" {}
+          disable_standby_reads = true
+
+    # KMS unseals the server, so the Shamir unseal sidecar and its Secret volume
+    # are not used. Drop them.
+    extraContainers: []
+    volumes: []
+    volumeMounts: []

deploy/helm/openbao/helm/values.yaml

@@ -89,6 +89,32 @@ openbao:
 
     podManagementPolicy: Parallel
 
+    # Auto-unseal through a KMS or HSM seal (for example awskms, azurekeyvault,
+    # gcpckms, transit, or pkcs11). Default off, so the server uses a Shamir seal
+    # and the auto-unseal sidecar. When on, the init hook initializes with
+    # recovery keys instead of a Shamir unseal key and the empty unseal Secret is
+    # not created.
+    #
+    # Enabling this flag alone is not enough. The deployer must also, in their
+    # values overlay: add the matching seal stanza to server.ha.raft.config (for
+    # example a seal "awskms" block), set server.extraContainers to [] to drop
+    # the auto-unseal sidecar, and remove the unseal volume and volumeMount. See
+    # values-autounseal.yaml.example.
+    autoUnseal:
+      enabled: false
+      # Recovery keys replace unseal keys under an auto-unseal seal. They do not
+      # unseal; they authorize root regeneration and rekey.
+      #
+      # Required post-init step. The init hook writes all shares into one Secret,
+      # <statefulset>-recovery-keys, so anyone who can read that Secret holds the
+      # full quorum and the threshold below protects nothing against them. After
+      # init, distribute the shares to separate custodians and delete the Secret;
+      # the hook logs the same reminder. Until that is done the threshold is
+      # advisory only.
+      recovery:
+        shares: 5
+        threshold: 3
+
     # enable HA
     ha:
       enabled: true