From 1f25bd65c2c5217cd194bbb157a2bf74748c50be Mon Sep 17 00:00:00 2001 From: Sunil Thorat Date: Tue, 6 Oct 2026 15:17:13 +0530 Subject: [PATCH 1/6] feat(openbao): support KMS auto-unseal in the self-managed chart Add an opt-in auto-unseal path to the bundled OpenBao chart next to the default single-share Shamir seal. It works with any OpenBao auto-unseal seal (awskms, azurekeyvault, gcpckms, transit, pkcs11). When server.autoUnseal.enabled is set: - the post-install init hook initializes with recovery keys (-recovery-shares / -recovery-threshold) instead of a Shamir unseal key, and stores them in a -recovery-keys Secret for export to a break-glass store; - the pre-install empty unseal Secret is not created; - the init script reads the seal type from `bao status` and skips the manual unseal and raft join, which the seal and retry_join handle. A deployer completes the setup in a values overlay: add the seal stanza to server.ha.raft.config, set server.extraContainers to [] to drop the auto-unseal sidecar, and clear the unseal volume. See helm/values-autounseal.yaml.example (AWS KMS shown as the example). Default behavior is unchanged: Shamir seal with the auto-unseal sidecar. Verified with helm template for both paths. Revoking the stored root token after bootstrap and migrating an existing Shamir cluster to an auto-unseal seal are follow-ups; this change keeps the root-token Secret and covers fresh installs. Co-Authored-By: Claude Opus 4.8 --- deploy/helm/openbao/helm/scripts/deploy.sh | 167 +++++++++++++++--- .../templates/hook-post-01-initcluster.yaml | 9 + .../templates/hook-pre-01-unseal-secret.yaml | 4 + .../helm/values-autounseal.yaml.example | 84 +++++++++ deploy/helm/openbao/helm/values.yaml | 19 ++ 5 files changed, 262 insertions(+), 21 deletions(-) create mode 100644 deploy/helm/openbao/helm/values-autounseal.yaml.example diff --git a/deploy/helm/openbao/helm/scripts/deploy.sh b/deploy/helm/openbao/helm/scripts/deploy.sh index c4fe17b62d..9425fdb41f 100755 --- a/deploy/helm/openbao/helm/scripts/deploy.sh +++ b/deploy/helm/openbao/helm/scripts/deploy.sh @@ -59,6 +59,58 @@ get_root_token() { kubectl get secret ${statefulset}-root-token -n ${namespace} -o jsonpath='{.data.root_token}' | base64 -d } +# Resolve the seal mode to use for init and unseal, into the global +# RESOLVED_SEAL_MODE ("auto" or "shamir"). Returns non-zero (fail closed) on an +# unreadable status or a mismatch, rather than guessing a path that could +# discard the only copy of a key. +# +# AUTO_UNSEAL is the chart's declared intent (server.autoUnseal.enabled), which +# also gates whether the unseal Secret exists. `bao status` reports the server's +# actual seal: recovery_seal=true for an auto-unseal (KMS or HSM) seal such as +# awskms, azurekeyvault, gcpckms, transit, or pkcs11; false for Shamir. These +# must agree: a flag set without the matching seal stanza (or the reverse) would +# otherwise take a path whose Secret does not exist. +# +# `bao status` exits 0 when unsealed and 2 when sealed; both return valid JSON. +# Any other exit (1) is a real error. jq -r is used, not jq -e, because a valid +# `false` makes jq -e exit non-zero. +RESOLVED_SEAL_MODE="" +resolve_seal_mode() { + local namespace=$1 + local statefulset=$2 + local declared="${AUTO_UNSEAL:-false}" + + local out rc + out=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + bao status -format=json 2>/dev/null) + rc=$? + if [ "${rc}" != "0" ] && [ "${rc}" != "2" ]; then + log_error "Could not read seal status from ${statefulset}-0 (bao status exit ${rc})" + return 1 + fi + + # Missing or false recovery_seal both mean "not auto-unseal" (Shamir); only + # an explicit true selects the auto path. `// false` keeps an uninitialized + # server whose status omits the field on the Shamir path, matching the + # pre-existing default behavior. + local server_mode="shamir" + if [ "$(printf '%s' "${out}" | jq -r '.recovery_seal // false' 2>/dev/null)" = "true" ]; then + server_mode="auto" + fi + + if [ "${declared}" = "true" ] && [ "${server_mode}" != "auto" ]; then + log_error "server.autoUnseal.enabled is set but the server reports no auto-unseal seal. Add the seal stanza to server.ha.raft.config (see values-autounseal.yaml.example)." + return 1 + fi + if [ "${declared}" != "true" ] && [ "${server_mode}" = "auto" ]; then + log_error "The server reports an auto-unseal seal but server.autoUnseal.enabled is not set. Enable it so the unseal Secret and init path match the seal." + return 1 + fi + + RESOLVED_SEAL_MODE="${server_mode}" + return 0 +} + # Runtime version-skew check: verify that the auto-unseal-sidecar container's # image tag matches the openbao server container's image tag in the live # StatefulSet spec. This complements the template-time guard in @@ -188,29 +240,78 @@ initialize_cluster() { log_info "All OpenBao pods are ready" log_info "Initializing OpenBao cluster" - local init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ - bao operator init \ - -key-shares=1 \ - -key-threshold=1 \ - -format=json) - - # Extract keys - local unseal_key=$(echo ${init_output} | jq -r '.unseal_keys_b64[0]') - local root_token=$(echo ${init_output} | jq -r '.root_token') - - # Check if unseal key is empty - if [ -z "${unseal_key}" ]; then - log_error "Failed to get unseal key from initialization output" + if ! resolve_seal_mode "${namespace}" "${statefulset}"; then return 1 fi - - # Update the secret with the new unseal key - kubectl patch secret ${statefulset}-unseal \ - --patch "data: + local init_output + local root_token + if [ "${RESOLVED_SEAL_MODE}" = "auto" ]; then + # Auto-unseal seal: initialize with recovery keys, not an unseal key. + # Recovery keys regenerate the root token and rekey; they never unseal + # (the seal does that), so no unseal key or unseal Secret is needed. + local recovery_shares="${RECOVERY_SHARES:-5}" + local recovery_threshold="${RECOVERY_THRESHOLD:-3}" + log_info "Auto-unseal seal detected; initializing with ${recovery_shares} recovery shares (threshold ${recovery_threshold})" + init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + bao operator init \ + -recovery-shares=${recovery_shares} \ + -recovery-threshold=${recovery_threshold} \ + -format=json) + root_token=$(echo ${init_output} | jq -r '.root_token') + if [ -z "${root_token}" ] || [ "${root_token}" = "null" ]; then + log_error "Failed to get root token from initialization output" + return 1 + fi + # Persist the recovery keys before anything else can discard them: once + # the server is initialized, `operator init` cannot reproduce them, so a + # failed write here would lose the only copy and make break-glass + # recovery impossible. Retry, and fail the init if it cannot be stored + # rather than reporting success without keys. + local recovery_keys + recovery_keys=$(echo ${init_output} | jq -c '.recovery_keys_b64') + if [ -z "${recovery_keys}" ] || [ "${recovery_keys}" = "null" ]; then + log_error "Initialization did not return recovery keys (.recovery_keys_b64 is null). Aborting." + return 1 + fi + local persisted=false + for attempt in 1 2 3; do + if kubectl create secret generic ${statefulset}-recovery-keys \ + -n ${namespace} \ + --from-literal=recovery_keys_b64="${recovery_keys}" \ + --dry-run=client -o yaml | kubectl apply -f -; then + persisted=true + break + fi + log_warn "Could not persist recovery keys (attempt ${attempt}/3); retrying..." + sleep 3 + done + if [ "${persisted}" != "true" ]; then + log_error "Failed to store recovery keys in Secret '${statefulset}-recovery-keys'. The server is initialized but the keys are not saved, so break-glass recovery is impossible. Tear down with cleanup.sh and reinitialize." + return 1 + fi + log_warn "Recovery keys stored in Secret '${statefulset}-recovery-keys'. Export them to a break-glass store and delete this Secret." + else + # Shamir seal: a single unseal key, stored for the auto-unseal sidecar. + init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + bao operator init \ + -key-shares=1 \ + -key-threshold=1 \ + -format=json) + local unseal_key=$(echo ${init_output} | jq -r '.unseal_keys_b64[0]') + root_token=$(echo ${init_output} | jq -r '.root_token') + if [ -z "${unseal_key}" ] || [ "${unseal_key}" = "null" ]; then + log_error "Failed to get unseal key from initialization output" + return 1 + fi + if ! kubectl patch secret ${statefulset}-unseal \ + --patch "data: unseal_key: $(echo -n "${unseal_key}" | base64)" \ - -n ${namespace} - - log_info "Updated Kubernetes secret '${statefulset}-unseal' with unseal key" + -n ${namespace}; then + log_error "Failed to store unseal key in Secret '${statefulset}-unseal'. The server is initialized but the unseal key is not saved; tear down with cleanup.sh and reinitialize." + return 1 + fi + log_info "Updated Kubernetes secret '${statefulset}-unseal' with unseal key" + fi # Store root token in a new secret log_info "Creating secret '${statefulset}-root-token' with root token..." @@ -229,10 +330,34 @@ get_unseal_key() { unseal_cluster() { local namespace=$1 local statefulset=$2 - local unseal_key=$(get_unseal_key "${namespace}" "${statefulset}") log_section "Unsealing OpenBao cluster" + if ! resolve_seal_mode "${namespace}" "${statefulset}"; then + return 1 + fi + if [ "${RESOLVED_SEAL_MODE}" = "auto" ]; then + # The auto-unseal seal unseals each node on start, and retry_join in the + # raft config joins the peers. No manual unseal or raft join is needed; + # wait for every pod to report unsealed. + log_info "Auto-unseal seal detected; waiting for all pods to unseal" + local end=$((SECONDS + 120)) + for i in {0..2}; do + while [ "$(kubectl exec ${statefulset}-${i} -c openbao -n ${namespace} -- bao status -format=json 2>/dev/null | jq -r '.sealed' 2>/dev/null)" != "false" ]; do + if [ $SECONDS -gt $end ]; then + log_error "Timeout waiting for pod ${statefulset}-${i} to auto-unseal" + return 1 + fi + log_info "Waiting for ${statefulset}-${i} to auto-unseal..." + sleep 5 + done + done + log_info "All pods auto-unsealed" + return 0 + fi + + local unseal_key=$(get_unseal_key "${namespace}" "${statefulset}") + # First unseal the primary node (pod 0) log_info "Unsealing primary pod ${statefulset}-0" if ! kubectl exec ${statefulset}-0 -n ${namespace} -- \ diff --git a/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml b/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml index 925ac10ce0..dfb61911af 100644 --- a/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml +++ b/deploy/helm/openbao/helm/templates/hook-post-01-initcluster.yaml @@ -52,6 +52,15 @@ spec: args: - -c - /scripts/deploy.sh {{ include "nvcf-openbao.namespace" . }} {{ $serverFullname }} helm + env: + - name: AUTO_UNSEAL + value: {{ .Values.openbao.server.autoUnseal.enabled | quote }} + {{- if .Values.openbao.server.autoUnseal.enabled }} + - name: RECOVERY_SHARES + value: {{ .Values.openbao.server.autoUnseal.recovery.shares | quote }} + - name: RECOVERY_THRESHOLD + value: {{ .Values.openbao.server.autoUnseal.recovery.threshold | quote }} + {{- end }} volumeMounts: - name: init-script readOnly: true diff --git a/deploy/helm/openbao/helm/templates/hook-pre-01-unseal-secret.yaml b/deploy/helm/openbao/helm/templates/hook-pre-01-unseal-secret.yaml index 23ed56bb38..30607d17ed 100644 --- a/deploy/helm/openbao/helm/templates/hook-pre-01-unseal-secret.yaml +++ b/deploy/helm/openbao/helm/templates/hook-pre-01-unseal-secret.yaml @@ -14,6 +14,9 @@ # limitations under the License. {{- $serverFullname := include "nvcf-openbao.serverFullname" . }} +{{- /* KMS auto-unseal does not use a Shamir unseal key, so skip the empty + unseal Secret when server.autoUnseal.enabled is set. */ -}} +{{- if not .Values.openbao.server.autoUnseal.enabled }} apiVersion: v1 kind: Secret metadata: @@ -26,3 +29,4 @@ metadata: type: Opaque data: unseal_key: "" +{{- end }} diff --git a/deploy/helm/openbao/helm/values-autounseal.yaml.example b/deploy/helm/openbao/helm/values-autounseal.yaml.example new file mode 100644 index 0000000000..990058642b --- /dev/null +++ b/deploy/helm/openbao/helm/values-autounseal.yaml.example @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Example values overlay for OpenBao auto-unseal. Apply on top of values.yaml. +# +# This turns on server.autoUnseal, adds a seal stanza so each node unseals +# through the seal on start, and drops the Shamir auto-unseal sidecar and its +# unseal Secret volume. The init hook then initializes with recovery keys. +# +# The seal block below uses AWS KMS as a concrete example. For Azure Key Vault, +# GCP KMS, HashiCorp/OpenBao Transit, or a PKCS#11 HSM, replace just the seal +# stanza with that provider's block (see the OpenBao seal documentation); the +# rest of this overlay is identical. +# +# The server pod needs the provider's unwrap permission on the key, granted out +# of band. For AWS KMS that is kms:Encrypt, kms:Decrypt and kms:DescribeKey, for +# example through an EKS Pod Identity or IRSA role. Automatic key rotation is +# safe where the provider keeps the key id and can still decrypt the previously +# wrapped root key. + +openbao: + server: + autoUnseal: + enabled: true + recovery: + shares: 5 + threshold: 3 + + # Override the raft config to add the seal stanza. Keep the rest in sync + # with values.yaml; only the seal block is new here. + ha: + raft: + config: | + ui = true + storage "raft" { + path = "/openbao/data/" + retry_join { + leader_api_addr = "http://openbao-server-0.openbao-server-internal:8200" + } + retry_join { + leader_api_addr = "http://openbao-server-1.openbao-server-internal:8200" + } + retry_join { + leader_api_addr = "http://openbao-server-2.openbao-server-internal:8200" + } + } + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + + custom_response_headers { + "default" = { + "X-Custom-Header" = ["HOSTNAME"], + }, + } + } + # Example: AWS KMS. Swap this block for azurekeyvault, gcpckms, + # transit, or pkcs11 to use a different provider. + seal "awskms" { + region = "" + kms_key_id = "" + } + plugin_directory = "/openbao/plugins/" + service_registration "kubernetes" {} + disable_standby_reads = true + + # KMS unseals the server, so the Shamir unseal sidecar and its Secret volume + # are not used. Drop them. + extraContainers: [] + volumes: [] + volumeMounts: [] diff --git a/deploy/helm/openbao/helm/values.yaml b/deploy/helm/openbao/helm/values.yaml index a71d9caf25..6d255a50d4 100644 --- a/deploy/helm/openbao/helm/values.yaml +++ b/deploy/helm/openbao/helm/values.yaml @@ -79,6 +79,25 @@ openbao: podManagementPolicy: Parallel + # Auto-unseal through a KMS or HSM seal (for example awskms, azurekeyvault, + # gcpckms, transit, or pkcs11). Default off, so the server uses a Shamir seal + # and the auto-unseal sidecar. When on, the init hook initializes with + # recovery keys instead of a Shamir unseal key and the empty unseal Secret is + # not created. + # + # Enabling this flag alone is not enough. The deployer must also, in their + # values overlay: add the matching seal stanza to server.ha.raft.config (for + # example a seal "awskms" block), set server.extraContainers to [] to drop + # the auto-unseal sidecar, and remove the unseal volume and volumeMount. See + # values-autounseal.yaml.example. + autoUnseal: + enabled: false + # Recovery keys replace unseal keys under an auto-unseal seal. They do not + # unseal; they authorize root regeneration and rekey. + recovery: + shares: 5 + threshold: 3 + # enable HA ha: enabled: true From 4c21d8522d50a452cba5eb39a375311ce7458f98 Mon Sep 17 00:00:00 2001 From: Sunil Thorat Date: Tue, 6 Oct 2026 17:35:31 +0530 Subject: [PATCH 2/6] fix(openbao): fail closed on unparseable seal status, add resolver test resolve_seal_mode now rejects an unparseable or non-boolean bao status instead of silently treating it as Shamir; a missing recovery_seal field still maps to Shamir (the pre-existing default). Add a unit test that mocks kubectl and covers the mode decision, both mismatch directions, status read errors, and malformed/non-boolean status. Co-Authored-By: Claude Opus 4.8 --- deploy/helm/openbao/helm/scripts/deploy.sh | 19 +++-- .../openbao/tests/resolve-seal-mode-test.sh | 73 +++++++++++++++++++ 2 files changed, 87 insertions(+), 5 deletions(-) create mode 100755 deploy/helm/openbao/tests/resolve-seal-mode-test.sh diff --git a/deploy/helm/openbao/helm/scripts/deploy.sh b/deploy/helm/openbao/helm/scripts/deploy.sh index 9425fdb41f..ec18805b6b 100755 --- a/deploy/helm/openbao/helm/scripts/deploy.sh +++ b/deploy/helm/openbao/helm/scripts/deploy.sh @@ -89,12 +89,21 @@ resolve_seal_mode() { return 1 fi - # Missing or false recovery_seal both mean "not auto-unseal" (Shamir); only - # an explicit true selects the auto path. `// false` keeps an uninitialized - # server whose status omits the field on the Shamir path, matching the - # pre-existing default behavior. + # Parse recovery_seal, failing closed on unparseable or non-boolean status. + # A missing field stays on the Shamir path (matches the pre-existing default + # for an uninitialized server); only an explicit boolean true selects auto. + # jq -r (not -e) is used so a valid false does not look like a jq failure. + local recovery_seal + recovery_seal=$(printf '%s' "${out}" | jq -rs ' + if (length != 1) or ((.[0] | type) != "object") then error("invalid status") + elif (.[0] | has("recovery_seal") | not) then false + elif (.[0].recovery_seal | type) != "boolean" then error("recovery_seal not boolean") + else .[0].recovery_seal end' 2>/dev/null) || { + log_error "Could not parse seal status from ${statefulset}-0" + return 1 + } local server_mode="shamir" - if [ "$(printf '%s' "${out}" | jq -r '.recovery_seal // false' 2>/dev/null)" = "true" ]; then + if [ "${recovery_seal}" = "true" ]; then server_mode="auto" fi diff --git a/deploy/helm/openbao/tests/resolve-seal-mode-test.sh b/deploy/helm/openbao/tests/resolve-seal-mode-test.sh new file mode 100755 index 0000000000..d646f84c23 --- /dev/null +++ b/deploy/helm/openbao/tests/resolve-seal-mode-test.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Unit test for resolve_seal_mode() in helm/scripts/deploy.sh. It mocks the +# `kubectl exec ... bao status` call and checks that the resolver picks the +# right seal mode, fails closed on a chart/server mismatch, and fails closed on +# an unreadable or malformed status. No cluster is required. +# +# Run: bash deploy/helm/openbao/tests/resolve-seal-mode-test.sh + +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_SH="${SCRIPT_DIR}/../helm/scripts/deploy.sh" + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT + +# Fake kubectl: ignores args and emits MOCK_JSON with exit code MOCK_RC, which +# is how `kubectl exec ... -- bao status -format=json` is consumed. +cat >"${workdir}/kubectl" <<'MOCK' +#!/usr/bin/env bash +printf '%s' "${MOCK_JSON:-}" +exit "${MOCK_RC:-0}" +MOCK +chmod +x "${workdir}/kubectl" +export PATH="${workdir}:${PATH}" + +# Loggers are defined in log.sh; stub them so the test has no dependency and so +# resolve_seal_mode's log output does not interfere. +log_error() { :; } +log_info() { :; } +log_warn() { :; } + +# Load only the RESOLVED_SEAL_MODE global and the resolve_seal_mode function, +# not the script's top-level install flow. +eval "$(sed -n '/^RESOLVED_SEAL_MODE=""/,/^}$/p' "${DEPLOY_SH}")" + +pass=0 +fail=0 +check() { # description AUTO_UNSEAL MOCK_JSON MOCK_RC expect_rc expect_mode + local desc="$1" + export AUTO_UNSEAL="$2" MOCK_JSON="$3" MOCK_RC="$4" + local exp_rc="$5" exp_mode="$6" + RESOLVED_SEAL_MODE="" + resolve_seal_mode ns sts + local rc=$? + if [ "${rc}" = "${exp_rc}" ] && { [ "${exp_rc}" != "0" ] || [ "${RESOLVED_SEAL_MODE}" = "${exp_mode}" ]; }; then + echo "PASS: ${desc}" + pass=$((pass + 1)) + else + echo "FAIL: ${desc} -> rc=${rc} mode=${RESOLVED_SEAL_MODE:-n/a} (wanted rc=${exp_rc} mode=${exp_mode})" + fail=$((fail + 1)) + fi +} + +# description AUTO_UNSEAL status JSON rc want_rc want_mode +check "shamir server, flag unset (default)" "" '{"sealed":true,"recovery_seal":false}' 2 0 shamir +check "shamir server, flag=false" false '{"sealed":true,"recovery_seal":false}' 2 0 shamir +check "shamir server, field omitted" false '{"sealed":true}' 2 0 shamir +check "auto server, flag=true, unsealed" true '{"sealed":false,"recovery_seal":true}' 0 0 auto +check "auto server, flag=true, sealed rc2" true '{"sealed":true,"recovery_seal":true}' 2 0 auto +check "mismatch: flag=true, server shamir" true '{"recovery_seal":false}' 2 1 '' +check "mismatch: flag=false, server auto" false '{"recovery_seal":true}' 0 1 '' +check "status read error (exit 1)" true '' 1 1 '' +check "malformed status, fail closed" false 'not json' 2 1 '' +check "empty status body, fail closed" false '' 2 1 '' +check "non-boolean recovery_seal, fail" true '{"recovery_seal":"true"}' 0 1 '' + +echo "-----" +echo "pass=${pass} fail=${fail}" +[ "${fail}" -eq 0 ] From bbfbcfd0f5b248c09dede729b87707d86f202eb7 Mon Sep 17 00:00:00 2001 From: Sunil Thorat Date: Tue, 6 Oct 2026 17:42:56 +0530 Subject: [PATCH 3/6] fix(openbao): fail on bao operator init error before parsing output Both init branches now check the exit status of `bao operator init` and stop with a clear message instead of misreporting a missing key or token. A partial init that leaves the server initialized without captured keys is called out as a tear-down-and-reinitialize case. Co-Authored-By: Claude Opus 4.8 --- deploy/helm/openbao/helm/scripts/deploy.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/deploy/helm/openbao/helm/scripts/deploy.sh b/deploy/helm/openbao/helm/scripts/deploy.sh index ec18805b6b..013c2940e1 100755 --- a/deploy/helm/openbao/helm/scripts/deploy.sh +++ b/deploy/helm/openbao/helm/scripts/deploy.sh @@ -261,11 +261,14 @@ initialize_cluster() { local recovery_shares="${RECOVERY_SHARES:-5}" local recovery_threshold="${RECOVERY_THRESHOLD:-3}" log_info "Auto-unseal seal detected; initializing with ${recovery_shares} recovery shares (threshold ${recovery_threshold})" - init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + if ! init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ bao operator init \ -recovery-shares=${recovery_shares} \ -recovery-threshold=${recovery_threshold} \ - -format=json) + -format=json); then + log_error "bao operator init failed. If the server is now initialized its keys were not captured; tear down with cleanup.sh and reinitialize." + return 1 + fi root_token=$(echo ${init_output} | jq -r '.root_token') if [ -z "${root_token}" ] || [ "${root_token}" = "null" ]; then log_error "Failed to get root token from initialization output" @@ -301,11 +304,14 @@ initialize_cluster() { log_warn "Recovery keys stored in Secret '${statefulset}-recovery-keys'. Export them to a break-glass store and delete this Secret." else # Shamir seal: a single unseal key, stored for the auto-unseal sidecar. - init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ + if ! init_output=$(kubectl exec ${statefulset}-0 -c openbao -n ${namespace} -- \ bao operator init \ -key-shares=1 \ -key-threshold=1 \ - -format=json) + -format=json); then + log_error "bao operator init failed. If the server is now initialized its keys were not captured; tear down with cleanup.sh and reinitialize." + return 1 + fi local unseal_key=$(echo ${init_output} | jq -r '.unseal_keys_b64[0]') root_token=$(echo ${init_output} | jq -r '.root_token') if [ -z "${unseal_key}" ] || [ "${unseal_key}" = "null" ]; then From c77041275814d12f0eeb7d7172bebabe602420c9 Mon Sep 17 00:00:00 2001 From: Sunil Thorat Date: Tue, 6 Oct 2026 21:52:41 +0530 Subject: [PATCH 4/6] docs(openbao): clarify recovery-key custody and PKCS#11 prerequisites Address PR review on the auto-unseal docs. - values.yaml: note the init hook stores all recovery shares in one Secret, so the shares/threshold protect nothing until an operator splits them to separate custodians and deletes the Secret. Call it a required post-init step next to the threshold. - values-autounseal.yaml.example: spell out that required post-install step with commands, and document that a PKCS#11 HSM needs the vendor PKCS#11 library plus either an HSM-enabled OpenBao build (cgo) or the openbao-plugins PKCS#11 KMS provider plugin, not just a seal-stanza swap. Co-Authored-By: Claude Opus 4.8 --- .../helm/values-autounseal.yaml.example | 27 ++++++++++++++++--- deploy/helm/openbao/helm/values.yaml | 7 +++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/deploy/helm/openbao/helm/values-autounseal.yaml.example b/deploy/helm/openbao/helm/values-autounseal.yaml.example index 990058642b..e75763dc34 100644 --- a/deploy/helm/openbao/helm/values-autounseal.yaml.example +++ b/deploy/helm/openbao/helm/values-autounseal.yaml.example @@ -20,15 +20,36 @@ # unseal Secret volume. The init hook then initializes with recovery keys. # # The seal block below uses AWS KMS as a concrete example. For Azure Key Vault, -# GCP KMS, HashiCorp/OpenBao Transit, or a PKCS#11 HSM, replace just the seal -# stanza with that provider's block (see the OpenBao seal documentation); the -# rest of this overlay is identical. +# GCP KMS, or HashiCorp/OpenBao Transit, replace just the seal stanza with that +# provider's block (see the OpenBao seal documentation); the rest of this overlay +# is identical. +# +# A PKCS#11 HSM is not a drop-in stanza swap. The seal "pkcs11" block points at +# the HSM vendor's PKCS#11 library (lib, token_label, key_label), which must be +# present in the server container. PKCS#11 support itself comes from either an +# HSM-enabled OpenBao build with PKCS#11 compiled in via cgo, or the external +# PKCS#11 KMS provider plugin (openbao-plugins). The stock image here ships +# neither; see the OpenBao pkcs11 seal docs. # # The server pod needs the provider's unwrap permission on the key, granted out # of band. For AWS KMS that is kms:Encrypt, kms:Decrypt and kms:DescribeKey, for # example through an EKS Pod Identity or IRSA role. Automatic key rotation is # safe where the provider keeps the key id and can still decrypt the previously # wrapped root key. +# +# Required post-install step, before treating the cluster as production. The init +# hook stores all recovery shares in one Secret, -recovery-keys. +# While it exists, any reader of that Secret holds the full recovery quorum, so +# the shares/threshold below give no protection. Finish the install by exporting +# the shares, splitting them among separate custodians offline, and deleting the +# Secret: +# +# kubectl get secret -recovery-keys -n \ +# -o jsonpath='{.data.recovery_keys_b64}' | base64 -d # then split offline +# kubectl delete secret -recovery-keys -n +# +# Only after the delete does the threshold protect the quorum. deploy.sh logs the +# same reminder at the end of init. openbao: server: diff --git a/deploy/helm/openbao/helm/values.yaml b/deploy/helm/openbao/helm/values.yaml index 71785def01..cf5360d1ce 100644 --- a/deploy/helm/openbao/helm/values.yaml +++ b/deploy/helm/openbao/helm/values.yaml @@ -104,6 +104,13 @@ openbao: enabled: false # Recovery keys replace unseal keys under an auto-unseal seal. They do not # unseal; they authorize root regeneration and rekey. + # + # Required post-init step. The init hook writes all shares into one Secret, + # -recovery-keys, so anyone who can read that Secret holds the + # full quorum and the threshold below protects nothing against them. After + # init, distribute the shares to separate custodians and delete the Secret; + # the hook logs the same reminder. Until that is done the threshold is + # advisory only. recovery: shares: 5 threshold: 3 From e14c8e338adbada79c26f27a3373d3b7487dc479 Mon Sep 17 00:00:00 2001 From: Sunil Thorat Date: Tue, 6 Oct 2026 22:07:50 +0530 Subject: [PATCH 5/6] fix(openbao): delete recovery-keys Secret in cleanup.sh The auto-unseal init path creates -recovery-keys; teardown left it orphaned. Delete it alongside the unseal and root-token Secrets, with --ignore-not-found so Shamir installs are unaffected. Co-Authored-By: Claude Opus 4.8 --- deploy/helm/openbao/cleanup.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deploy/helm/openbao/cleanup.sh b/deploy/helm/openbao/cleanup.sh index caa2f1f154..69f069f045 100755 --- a/deploy/helm/openbao/cleanup.sh +++ b/deploy/helm/openbao/cleanup.sh @@ -39,12 +39,16 @@ cleanup_cluster() { fi log_success "Cleaned up OpenBao PVCs in namespace: $namespace" - # Delete the unseal key and root token secret + # Delete the unseal key, recovery keys, and root token secret log_info "Deleting secrets in namespace: $namespace" if ! kubectl delete secret $statefulset-unseal -n $namespace --ignore-not-found=true > /dev/null 2>&1; then log_error "Failed to delete unseal secret (exit code: $?): $?" exit 1 fi + if ! kubectl delete secret $statefulset-recovery-keys -n $namespace --ignore-not-found=true > /dev/null 2>&1; then + log_error "Failed to delete recovery-keys secret (exit code: $?): $?" + exit 1 + fi if ! kubectl delete secret $statefulset-root-token -n $namespace --ignore-not-found=true > /dev/null 2>&1; then log_error "Failed to delete root token secret (exit code: $?): $?" exit 1 From d9aed00984f659af471794778d17d01d63864542 Mon Sep 17 00:00:00 2001 From: Sunil Thorat Date: Tue, 6 Oct 2026 22:14:50 +0530 Subject: [PATCH 6/6] docs(openbao): don't assert the default image lacks PKCS#11 support The chart only defaults a server image tag and does not pin registry/repo, and whether a given build includes PKCS#11 is the deployer's to confirm. State the requirement and put it on the deployer instead. Co-Authored-By: Claude Opus 4.8 --- deploy/helm/openbao/helm/values-autounseal.yaml.example | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/helm/openbao/helm/values-autounseal.yaml.example b/deploy/helm/openbao/helm/values-autounseal.yaml.example index e75763dc34..a5b9ca524b 100644 --- a/deploy/helm/openbao/helm/values-autounseal.yaml.example +++ b/deploy/helm/openbao/helm/values-autounseal.yaml.example @@ -28,8 +28,9 @@ # the HSM vendor's PKCS#11 library (lib, token_label, key_label), which must be # present in the server container. PKCS#11 support itself comes from either an # HSM-enabled OpenBao build with PKCS#11 compiled in via cgo, or the external -# PKCS#11 KMS provider plugin (openbao-plugins). The stock image here ships -# neither; see the OpenBao pkcs11 seal docs. +# PKCS#11 KMS provider plugin (openbao-plugins). Make sure the OpenBao image you +# run provides one of these; the seal stanza alone does not. See the OpenBao +# pkcs11 seal docs. # # The server pod needs the provider's unwrap permission on the key, granted out # of band. For AWS KMS that is kms:Encrypt, kms:Decrypt and kms:DescribeKey, for