Polish REST SigV4 auth integration

Simplify AWS SDK wiring, remove premature session-cache plumbing, tighten credential validation, and strengthen SigV4 tests.

Author: wgtmac

.github/workflows/aws_test.yml

@@ -15,9 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# AWS-related tests. ICEBERG_S3 and ICEBERG_SIGV4 are exercised individually and
-# together; with both on, ICEBERG_AWSSDK_SOURCE defaults to BUNDLED so SigV4
-# reuses Arrow's bundled aws-cpp-sdk-core — a single AWS SDK is linked (no ODR).
 name: AWS Tests
 
 on:
@@ -50,33 +47,26 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - title: AMD64 Ubuntu 24.04, S3
+          - title: Ubuntu 24.04, S3 + SigV4, bundled AWS SDK
             runs-on: ubuntu-24.04
             CC: gcc-14
             CXX: g++-14
             s3: "ON"
-            sigv4: "OFF"
-          - title: AMD64 Ubuntu 24.04, SigV4
-            runs-on: ubuntu-24.04
-            CC: gcc-14
-            CXX: g++-14
-            s3: "OFF"
             sigv4: "ON"
-            aws-sdk-features: core
-          - title: AMD64 Ubuntu 24.04, S3 + SigV4
+            bundle_awssdk: "ON"
+          - title: Ubuntu 24.04, S3 + SigV4, system AWS SDK
             runs-on: ubuntu-24.04
             CC: gcc-14
             CXX: g++-14
             s3: "ON"
             sigv4: "ON"
-            # Arrow's S3 filesystem consumes this same AWS SDK, so it needs the
-            # S3-related components in addition to core (config is required by
-            # Arrow's FindAWSSDKAlt).
+            bundle_awssdk: "OFF"
             aws-sdk-features: core,config,s3,identity-management,sts,transfer
-          - title: AArch64 macOS 26, S3
+          - title: macOS 26 ARM64, S3, bundled AWS SDK
             runs-on: macos-26
             s3: "ON"
             sigv4: "OFF"
+            bundle_awssdk: "ON"
     env:
       ICEBERG_TEST_S3_URI: s3://iceberg-test
       AWS_ACCESS_KEY_ID: minio
@@ -94,14 +84,14 @@ jobs:
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
       - name: Cache vcpkg packages
-        if: ${{ matrix.sigv4 == 'ON' && matrix.s3 == 'OFF' }}
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         id: vcpkg-cache
         with:
           path: /usr/local/share/vcpkg/installed
           key: vcpkg-x64-linux-aws-sdk-cpp-s3-${{ matrix.s3 }}-sigv4-${{ matrix.sigv4 }}-${{ hashFiles('.github/workflows/aws_test.yml') }}
       - name: Install AWS SDK via vcpkg
-        if: ${{ matrix.sigv4 == 'ON' && matrix.s3 == 'OFF' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
         shell: bash
         # Retry to ride out transient GitHub/mirror download failures (504s).
         run: |
@@ -126,8 +116,8 @@ jobs:
       - name: Build and test Iceberg
         shell: bash
         env:
-          CMAKE_TOOLCHAIN_FILE: ${{ matrix.sigv4 == 'ON' && matrix.s3 == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
-        run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ${{ matrix.s3 }} ${{ matrix.sigv4 }}
+          CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
+        run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
 
   # Exercise the Meson build with SigV4 enabled (resolves aws-cpp-sdk-core via
   # its CMake config, not pkg-config whose Cflags force -std=c++11).

CMakeLists.txt

@@ -53,13 +53,8 @@ option(ICEBERG_SQL_SQLITE "Build the SQLite connector for the SQL catalog" OFF)
 option(ICEBERG_SQL_POSTGRESQL "Build the PostgreSQL connector for the SQL catalog" OFF)
 option(ICEBERG_SQL_MYSQL "Build the MySQL connector for the SQL catalog" OFF)
 option(ICEBERG_S3 "Build with S3 support" OFF)
-option(ICEBERG_SIGV4 "Build SigV4 authentication support (requires AWS SDK)" OFF)
-set(ICEBERG_AWSSDK_SOURCE
-    "AUTO"
-    CACHE STRING "AWS SDK source for SigV4: AUTO (reuse Arrow's bundled AWS SDK when \
-ICEBERG_S3 is ON, otherwise SYSTEM), SYSTEM (find an installed AWS SDK), or \
-BUNDLED (reuse Arrow's bundled AWS SDK; requires ICEBERG_S3)")
-set_property(CACHE ICEBERG_AWSSDK_SOURCE PROPERTY STRINGS AUTO SYSTEM BUNDLED)
+option(ICEBERG_SIGV4 "Build with SigV4 support" OFF)
+option(ICEBERG_BUNDLE_AWSSDK "Bundle AWS SDK for S3/SigV4 support" ON)
 option(ICEBERG_ENABLE_ASAN "Enable Address Sanitizer" OFF)
 option(ICEBERG_ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)
 
@@ -83,12 +78,6 @@ if(ICEBERG_BUILD_REST_INTEGRATION_TESTS AND WIN32)
   message(WARNING "Cannot build rest integration test on Windows, turning it off.")
 endif()
 
-# ICEBERG_S3 requires ICEBERG_BUILD_BUNDLE
-if(NOT ICEBERG_BUILD_BUNDLE AND ICEBERG_S3)
-  set(ICEBERG_S3 OFF)
-  message(STATUS "ICEBERG_S3 is disabled because ICEBERG_BUILD_BUNDLE is OFF")
-endif()
-
 include(CMakeParseArguments)
 include(IcebergBuildUtils)
 include(IcebergSanitizer)

ci/scripts/build_iceberg.sh

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Usage: build_iceberg.sh <source_dir> [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] [sigv4=OFF]
+# Usage: build_iceberg.sh <source_dir> [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] [sigv4=OFF] [bundle_awssdk=ON]
 
 set -eux
 
@@ -27,6 +27,7 @@ build_rest_integration_test=${2:-OFF}
 build_enable_sccache=${3:-OFF}
 build_enable_s3=${4:-OFF}
 build_enable_sigv4=${5:-OFF}
+build_bundle_awssdk=${6:-ON}
 run_tests=${ICEBERG_RUN_TESTS:-ON}
 
 mkdir ${build_dir}
@@ -56,12 +57,17 @@ else
     CMAKE_ARGS+=("-DICEBERG_SIGV4=OFF")
 fi
 
+if [[ "${build_bundle_awssdk}" == "ON" ]]; then
+    CMAKE_ARGS+=("-DICEBERG_BUNDLE_AWSSDK=ON")
+else
+    CMAKE_ARGS+=("-DICEBERG_BUNDLE_AWSSDK=OFF")
+fi
+
 if is_windows; then
     CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
     CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
 else
     # Pass an externally provided toolchain (e.g. vcpkg for the SigV4 job)
-    # explicitly instead of relying on CMake >= 3.21 reading the env var.
     if [[ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]]; then
         CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}")
     fi

cmake_modules/IcebergThirdpartyToolchain.cmake

@@ -19,23 +19,49 @@
 # third party libraries.
 set(ICEBERG_SYSTEM_DEPENDENCIES)
 set(ICEBERG_ARROW_INSTALL_INTERFACE_LIBS)
-
-if(ICEBERG_SIGV4)
-  set(ICEBERG_AWSSDK_SOURCE_RESOLVED "${ICEBERG_AWSSDK_SOURCE}")
-  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "AUTO")
-    if(ICEBERG_S3)
-      set(ICEBERG_AWSSDK_SOURCE_RESOLVED "BUNDLED")
-    else()
-      set(ICEBERG_AWSSDK_SOURCE_RESOLVED "SYSTEM")
-    endif()
+set(ICEBERG_AWSSDK_BUNDLED FALSE)
+if(ICEBERG_S3 AND ICEBERG_BUNDLE_AWSSDK)
+  if(NOT ICEBERG_BUILD_BUNDLE)
+    message(FATAL_ERROR "ICEBERG_BUNDLE_AWSSDK requires ICEBERG_BUILD_BUNDLE to be ON")
   endif()
-  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "BUNDLED" AND NOT ICEBERG_S3)
-    message(FATAL_ERROR "ICEBERG_AWSSDK_SOURCE=BUNDLED requires ICEBERG_S3=ON: "
-                        "the bundled AWS SDK is provided by Arrow's S3 support.")
+  set(ICEBERG_AWSSDK_BUNDLED TRUE)
+endif()
+
+set(ICEBERG_AWSSDK_COMPONENTS)
+if(NOT ICEBERG_AWSSDK_BUNDLED)
+  if(ICEBERG_S3)
+    list(APPEND
+         ICEBERG_AWSSDK_COMPONENTS
+         core
+         config
+         s3
+         transfer
+         identity-management
+         sts)
+  elseif(ICEBERG_SIGV4)
+    list(APPEND ICEBERG_AWSSDK_COMPONENTS core)
   endif()
-  message(STATUS "AWS SDK source for SigV4: ${ICEBERG_AWSSDK_SOURCE_RESOLVED}")
 endif()
 
+# ----------------------------------------------------------------------
+# AWS SDK for C++
+
+function(resolve_aws_sdk_dependency)
+  if(NOT ICEBERG_AWSSDK_COMPONENTS)
+    return()
+  endif()
+  find_package(AWSSDK REQUIRED COMPONENTS ${ICEBERG_AWSSDK_COMPONENTS})
+  list(APPEND ICEBERG_SYSTEM_DEPENDENCIES AWSSDK)
+  set(ICEBERG_SYSTEM_DEPENDENCIES
+      ${ICEBERG_SYSTEM_DEPENDENCIES}
+      PARENT_SCOPE)
+  # Forwarded to find_dependency(AWSSDK ...) in iceberg-config.cmake.in so
+  # downstream installed builds load the same AWS SDK targets.
+  set(ICEBERG_FIND_EXTRA_ARGS_AWSSDK
+      "COMPONENTS;${ICEBERG_AWSSDK_COMPONENTS}"
+      PARENT_SCOPE)
+endfunction()
+
 # ----------------------------------------------------------------------
 # Versions and URLs for toolchain builds
 #
@@ -126,12 +152,10 @@ function(resolve_arrow_dependency)
   set(ARROW_RUNTIME_SIMD_LEVEL "NONE")
   set(ARROW_POSITION_INDEPENDENT_CODE ON)
   set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
-  if(ICEBERG_S3
-     AND ICEBERG_SIGV4
-     AND ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "SYSTEM")
+  set(ARROW_WITH_ZLIB ON)
+  if(ICEBERG_S3 AND NOT ICEBERG_AWSSDK_BUNDLED)
     set(AWSSDK_SOURCE "SYSTEM")
   endif()
-  set(ARROW_WITH_ZLIB ON)
   set(ZLIB_SOURCE "SYSTEM")
   set(ARROW_VERBOSE_THIRDPARTY_BUILD OFF)
   set(CMAKE_CXX_STANDARD 20)
@@ -641,6 +665,13 @@ resolve_nanoarrow_dependency()
 resolve_croaring_dependency()
 resolve_nlohmann_json_dependency()
 
+if(ICEBERG_S3 OR ICEBERG_SIGV4)
+  if(ICEBERG_SIGV4 AND NOT ICEBERG_BUILD_REST)
+    message(FATAL_ERROR "ICEBERG_SIGV4 requires ICEBERG_BUILD_REST to be ON")
+  endif()
+  resolve_aws_sdk_dependency()
+endif()
+
 if(ICEBERG_BUILD_BUNDLE)
   resolve_arrow_dependency()
   resolve_avro_dependency()
@@ -654,30 +685,3 @@ endif()
 if(ICEBERG_BUILD_SQL_CATALOG)
   resolve_sql_catalog_dependencies()
 endif()
-
-# ----------------------------------------------------------------------
-# AWS SDK for C++
-
-function(resolve_aws_sdk_dependency)
-  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "BUNDLED")
-    message(STATUS "SigV4 reuses Arrow's bundled AWS SDK (aws-cpp-sdk-core)")
-    return()
-  endif()
-  find_package(AWSSDK REQUIRED COMPONENTS core)
-  list(APPEND ICEBERG_SYSTEM_DEPENDENCIES AWSSDK)
-  set(ICEBERG_SYSTEM_DEPENDENCIES
-      ${ICEBERG_SYSTEM_DEPENDENCIES}
-      PARENT_SCOPE)
-  # Forwarded to find_dependency(AWSSDK ...) in iceberg-config.cmake.in so
-  # downstream installed builds load aws-cpp-sdk-core via AWSSDK_FIND_COMPONENTS.
-  set(ICEBERG_FIND_EXTRA_ARGS_AWSSDK
-      "COMPONENTS;core"
-      PARENT_SCOPE)
-endfunction()
-
-if(ICEBERG_SIGV4)
-  if(NOT ICEBERG_BUILD_REST)
-    message(FATAL_ERROR "ICEBERG_SIGV4 requires ICEBERG_BUILD_REST to be ON")
-  endif()
-  resolve_aws_sdk_dependency()
-endif()

meson.options

@@ -44,8 +44,6 @@ option(
     value: 'disabled',
 )
 
-# Resolves a system-installed AWS SDK via its CMake config; the bundled-AWS
-# path (ICEBERG_AWSSDK_SOURCE=BUNDLED) is CMake-only.
 option(
     'sigv4',
     type: 'feature',

src/iceberg/catalog/rest/CMakeLists.txt

@@ -23,13 +23,12 @@ set(ICEBERG_REST_SOURCES
     auth/auth_properties.cc
     auth/auth_session.cc
     auth/oauth2_util.cc
-    auth/sigv4_auth_manager.cc
+    auth/sigv4_manager.cc
     auth/token_refresh_scheduler.cc
     catalog_properties.cc
     endpoint.cc
     error_handlers.cc
     http_client.cc
-    http_request.cc
     json_serde.cc
     resource_paths.cc
     rest_catalog.cc
@@ -58,7 +57,7 @@ list(APPEND
 if(ICEBERG_SIGV4)
   list(APPEND ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS aws-cpp-sdk-core)
   list(APPEND ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS aws-cpp-sdk-core)
-  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "SYSTEM")
+  if(NOT ICEBERG_AWSSDK_BUNDLED)
     list(APPEND ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS aws-cpp-sdk-core)
     list(APPEND ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS aws-cpp-sdk-core)
   endif()
@@ -76,12 +75,16 @@ add_iceberg_lib(iceberg_rest
                 SHARED_INSTALL_INTERFACE_LIBS
                 ${ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS})
 
-if(ICEBERG_SIGV4)
-  foreach(LIB iceberg_rest_static iceberg_rest_shared)
-    if(TARGET ${LIB})
-      target_compile_definitions(${LIB} PUBLIC ICEBERG_SIGV4)
+foreach(LIB iceberg_rest_static iceberg_rest_shared)
+  if(TARGET ${LIB})
+    if(ICEBERG_SIGV4)
+      target_compile_definitions(${LIB}
+                                 PUBLIC "$<BUILD_INTERFACE:ICEBERG_SIGV4_ENABLED=1>")
+    else()
+      target_compile_definitions(${LIB}
+                                 PUBLIC "$<BUILD_INTERFACE:ICEBERG_SIGV4_ENABLED=0>")
     endif()
-  endforeach()
-endif()
+  endif()
+endforeach()
 
 iceberg_install_all_headers(iceberg/catalog/rest)

src/iceberg/catalog/rest/auth/auth_manager.cc

@@ -38,7 +38,8 @@ Result<std::shared_ptr<AuthSession>> AuthManager::InitSession(
 }
 
 Result<std::shared_ptr<AuthSession>> AuthManager::ContextualSession(
-    [[maybe_unused]] const SessionContext& context, std::shared_ptr<AuthSession> parent) {
+    [[maybe_unused]] const std::unordered_map<std::string, std::string>& context,
+    std::shared_ptr<AuthSession> parent) {
   // By default, return the parent session as-is
   return parent;
 }

src/iceberg/catalog/rest/auth/auth_manager.h

@@ -23,7 +23,6 @@
 #include <string>
 #include <unordered_map>
 
-#include "iceberg/catalog/rest/auth/session_context.h"
 #include "iceberg/catalog/rest/iceberg_rest_export.h"
 #include "iceberg/catalog/rest/type_fwd.h"
 #include "iceberg/result.h"
@@ -71,12 +70,13 @@ class ICEBERG_REST_EXPORT AuthManager {
   /// This method is used by SessionCatalog to create sessions for different contexts
   /// (e.g., different users or tenants).
   ///
-  /// \param context Per-session properties and credentials.
+  /// \param context Context properties (e.g., user credentials, tenant info).
   /// \param parent Catalog session to inherit from or return as-is.
   /// \return A context-specific session, or the parent session if no context-specific
   /// session is needed, or an error if session creation fails.
   virtual Result<std::shared_ptr<AuthSession>> ContextualSession(
-      const SessionContext& context, std::shared_ptr<AuthSession> parent);
+      const std::unordered_map<std::string, std::string>& context,
+      std::shared_ptr<AuthSession> parent);
 
   /// \brief Create or reuse a session scoped to a single table/view.
   ///

src/iceberg/catalog/rest/auth/auth_properties.h

@@ -59,6 +59,10 @@ class ICEBERG_REST_EXPORT AuthProperties : public ConfigBase<AuthProperties> {
   inline static const std::string kSigV4Enabled = "rest.sigv4-enabled";
   inline static const std::string kSigV4DelegateAuthType =
       "rest.auth.sigv4.delegate-auth-type";
+
+  /// SigV4 signing region. If unset, SigV4 resolves the signing region from
+  /// AWS environment/profile configuration and fails if no region can be
+  /// resolved.
   inline static const std::string kSigV4SigningRegion = "rest.signing-region";
   inline static const std::string kSigV4SigningName = "rest.signing-name";
   inline static const std::string kSigV4SigningNameDefault = "execute-api";

src/iceberg/catalog/rest/auth/auth_session.cc

@@ -43,12 +43,11 @@ class DefaultAuthSession : public AuthSession {
   explicit DefaultAuthSession(std::unordered_map<std::string, std::string> headers)
       : headers_(std::move(headers)) {}
 
-  Result<HttpRequest> Authenticate(const HttpRequest& request) override {
-    HttpRequest authenticated = request;
+  Result<HttpRequest> Authenticate(HttpRequest request) override {
     for (const auto& [key, value] : headers_) {
-      authenticated.headers.try_emplace(key, value);
+      request.headers.try_emplace(key, value);
     }
-    return authenticated;
+    return request;
   }
 
  private:
@@ -78,13 +77,12 @@ class OAuth2AuthSession : public AuthSession,
     return session;
   }
 
-  Result<HttpRequest> Authenticate(const HttpRequest& request) override {
-    HttpRequest authenticated = request;
+  Result<HttpRequest> Authenticate(HttpRequest request) override {
     std::shared_lock lock(mutex_);
     for (const auto& [key, value] : headers_) {
-      authenticated.headers.try_emplace(key, value);
+      request.headers.try_emplace(key, value);
     }
-    return authenticated;
+    return request;
   }
 
   Status Close() override { return CloseImpl(); }