Skip to content

Commit 1b6d6a3

Browse files
shangxinliclaude
andauthored
build: enable Parquet modular encryption and zstd in vendored Arrow (#989)
Part of #988. Adds the `ICEBERG_ENCRYPTION` CMake option (default on), which builds the vendored Arrow with Parquet modular encryption and zstd and links OpenSSL into the bundle library. This makes the crypto available for the following table encryption steps; there is no behavior change for tables. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 26ae9ea commit 1b6d6a3

11 files changed

Lines changed: 180 additions & 12 deletions

File tree

‎.github/workflows/aws_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ jobs:
8787
- name: Install dependencies on Ubuntu
8888
if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
8989
shell: bash
90-
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev
90+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev libjitterentropy3-dev
9191
- name: Cache vcpkg packages
9292
if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}
9393
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0

‎.github/workflows/cpp-linter.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
shell: bash
6161
run: |
6262
sudo apt-get update
63-
sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev
63+
sudo apt-get install -y libcurl4-openssl-dev libssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev
6464
- name: Set up sccache
6565
uses: ./.github/actions/setup-sccache
6666
with:

‎.github/workflows/rc.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ jobs:
104104
- name: Install dependencies on Ubuntu
105105
if: ${{ startsWith(matrix.os, 'ubuntu') }}
106106
shell: bash
107-
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
107+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev
108108

109109
- name: Verify
110110
run: |

‎.github/workflows/sanitizer_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ jobs:
5151
persist-credentials: false
5252
- name: Install dependencies
5353
shell: bash
54-
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
54+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev
5555
- name: Set up sccache
5656
uses: ./.github/actions/setup-sccache
5757
with:

‎.github/workflows/sql_catalog_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ jobs:
9696
if: ${{ startsWith(matrix.runs-on, 'windows') && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
9797
shell: pwsh
9898
run: |
99-
vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
99+
vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows openssl:x64-windows
100100
- name: Set up sccache
101101
uses: ./.github/actions/setup-sccache
102102
with:

‎.github/workflows/test.yml‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@ jobs:
5353
matrix:
5454
include:
5555
- cmake_build_type: Debug
56-
extra_cmake_args: "-DICEBERG_CXX20_COMPAT=ON"
56+
extra_cmake_args: "-DICEBERG_CXX20_COMPAT=ON -DICEBERG_ENCRYPTION=ON"
5757
api_mode: C++20 Compat
5858
- cmake_build_type: RelWithDebInfo
59-
extra_cmake_args: ""
59+
extra_cmake_args: "-DICEBERG_ENCRYPTION=ON"
6060
api_mode: C++23
6161
steps:
6262
- name: Checkout iceberg-cpp
@@ -65,7 +65,7 @@ jobs:
6565
persist-credentials: false
6666
- name: Install dependencies
6767
shell: bash
68-
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
68+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev
6969
- name: Set up sccache
7070
uses: ./.github/actions/setup-sccache
7171
with:
@@ -130,7 +130,7 @@ jobs:
130130
persist-credentials: false
131131
- name: Install dependencies
132132
shell: bash
133-
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
133+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev
134134
- name: Restore sccache cache
135135
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
136136
with:
@@ -179,7 +179,8 @@ jobs:
179179
- name: Build Iceberg
180180
shell: bash
181181
env:
182-
ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON"
182+
# Enables the Homebrew keg-only OpenSSL lookup, which only runs on macOS.
183+
ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON -DICEBERG_ENCRYPTION=ON"
183184
run: |
184185
ci/scripts/build_iceberg.sh \
185186
--source-dir "$(pwd)" \
@@ -231,15 +232,15 @@ jobs:
231232
if: ${{ steps.vcpkg-cache.outputs.cache-hit != 'true' }}
232233
shell: pwsh
233234
run: |
234-
vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows
235+
vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows openssl:x64-windows
235236
- name: Set up sccache
236237
uses: ./.github/actions/setup-sccache
237238
with:
238239
key-prefix: sccache-test-windows
239240
- name: Build Iceberg
240241
shell: pwsh
241242
env:
242-
ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON"
243+
ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON -DICEBERG_ENCRYPTION=ON"
243244
run: |
244245
$ErrorActionPreference = "Stop"
245246
bash -lc '

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ option(ICEBERG_BUILD_SHARED "Build shared library" OFF)
5151
option(ICEBERG_BUILD_TESTS "Build tests" ON)
5252
option(ICEBERG_BUILD_BENCHMARKS "Build benchmarks" OFF)
5353
option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON)
54+
option(ICEBERG_ENCRYPTION
55+
"Build table encryption support (Parquet modular encryption, AES GCM streams)" OFF)
5456
option(ICEBERG_BUILD_REST "Build rest catalog client" ON)
5557
option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF)
5658
option(ICEBERG_BUILD_HIVE "Build hive (HMS) catalog client" OFF)

‎cmake_modules/IcebergThirdpartyToolchain.cmake‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,28 @@ endmacro()
163163
# ----------------------------------------------------------------------
164164
# Apache Arrow
165165

166+
# Find OpenSSL for table encryption. Called before Arrow is configured, so that the
167+
# imported OpenSSL targets are visible to the vendored Arrow subdirectories.
168+
macro(resolve_openssl_for_encryption)
169+
if(APPLE
170+
AND NOT DEFINED OPENSSL_ROOT_DIR
171+
AND NOT DEFINED ENV{OPENSSL_ROOT_DIR})
172+
# Homebrew's OpenSSL is keg-only and not on the default search path.
173+
execute_process(COMMAND brew --prefix openssl@3
174+
OUTPUT_VARIABLE ICEBERG_BREW_OPENSSL_PREFIX
175+
OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
176+
if(ICEBERG_BREW_OPENSSL_PREFIX AND EXISTS "${ICEBERG_BREW_OPENSSL_PREFIX}")
177+
set(OPENSSL_ROOT_DIR "${ICEBERG_BREW_OPENSSL_PREFIX}")
178+
endif()
179+
endif()
180+
find_package(OpenSSL REQUIRED)
181+
endmacro()
182+
166183
function(resolve_arrow_dependency)
184+
if(ICEBERG_ENCRYPTION)
185+
resolve_openssl_for_encryption()
186+
endif()
187+
167188
prepare_fetchcontent()
168189

169190
# Prevent Arrow from injecting -Werror into CMAKE_CXX_FLAGS_DEBUG via
@@ -178,12 +199,21 @@ function(resolve_arrow_dependency)
178199
set(ARROW_S3 ${ICEBERG_S3})
179200
set(ARROW_JSON ON)
180201
set(ARROW_PARQUET ON)
202+
if(ICEBERG_ENCRYPTION)
203+
# Parquet modular encryption, used for encrypted Iceberg data and delete files.
204+
set(PARQUET_REQUIRE_ENCRYPTION ON)
205+
endif()
181206
set(ARROW_ENABLE_THREADING ON)
182207
set(ARROW_SIMD_LEVEL "NONE")
183208
set(ARROW_RUNTIME_SIMD_LEVEL "NONE")
184209
set(ARROW_POSITION_INDEPENDENT_CODE ON)
185210
set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
186211
set(ARROW_WITH_ZLIB ON)
212+
# zstd is the default value of the write.parquet.compression-codec table property,
213+
# so a vendored Arrow without it cannot write Iceberg's default Parquet files.
214+
# Not required by modular encryption itself; parquet_encryption_test covers the
215+
# encrypted-plus-zstd combination to keep the two features verified together.
216+
set(ARROW_WITH_ZSTD ON)
187217
if(ICEBERG_S3 AND NOT ICEBERG_AWSSDK_BUNDLED)
188218
set(AWSSDK_SOURCE "SYSTEM")
189219
endif()
@@ -217,6 +247,25 @@ function(resolve_arrow_dependency)
217247
fetchcontent_makeavailable(VendoredArrow)
218248

219249
if(vendoredarrow_SOURCE_DIR)
250+
# Arrow's Parquet targets do not inherit the OpenSSL include directory when
251+
# Arrow is vendored; add it for the Parquet encryption sources. Depending on the
252+
# platform and build type, they are compiled in the object library or directly in
253+
# the static/shared library. The bundle library links OpenSSL::Crypto.
254+
#
255+
# Only vendored targets are touched here. With system Arrow the same names can
256+
# be ALIAS targets created by a parent project, and target_include_directories
257+
# cannot modify an ALIAS.
258+
if(ICEBERG_ENCRYPTION)
259+
foreach(parquet_target parquet_objlib parquet_static parquet_shared)
260+
if(TARGET ${parquet_target})
261+
get_target_property(parquet_target_alias ${parquet_target} ALIASED_TARGET)
262+
if(NOT parquet_target_alias)
263+
target_include_directories(${parquet_target} PRIVATE ${OPENSSL_INCLUDE_DIR})
264+
endif()
265+
endif()
266+
endforeach()
267+
endif()
268+
220269
if(NOT TARGET Arrow::arrow_static)
221270
add_library(Arrow::arrow_static INTERFACE IMPORTED)
222271
target_link_libraries(Arrow::arrow_static INTERFACE arrow_static)
@@ -268,6 +317,12 @@ function(resolve_arrow_dependency)
268317
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow Parquet)
269318
endif()
270319

320+
# The bundle exports OpenSSL::Crypto in its link interface for both vendored and
321+
# system Arrow, so consumers must find OpenSSL in both modes.
322+
if(ICEBERG_ENCRYPTION)
323+
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL)
324+
endif()
325+
271326
set(ICEBERG_SYSTEM_DEPENDENCIES
272327
${ICEBERG_SYSTEM_DEPENDENCIES}
273328
PARENT_SCOPE)

‎src/iceberg/CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,16 @@ if(ICEBERG_BUILD_BUNDLE)
360360
"$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>"
361361
)
362362

363+
if(ICEBERG_ENCRYPTION)
364+
foreach(libs
365+
ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS
366+
ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS
367+
ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS
368+
ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS)
369+
list(APPEND ${libs} OpenSSL::Crypto)
370+
endforeach()
371+
endif()
372+
363373
add_iceberg_lib(iceberg_bundle
364374
SOURCES
365375
${ICEBERG_BUNDLE_SOURCES}

‎src/iceberg/test/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,9 @@ if(ICEBERG_BUILD_BUNDLE)
221221
parquet_metrics_test.cc
222222
parquet_schema_test.cc
223223
parquet_test.cc)
224+
if(ICEBERG_ENCRYPTION)
225+
target_sources(parquet_test PRIVATE parquet_encryption_test.cc)
226+
endif()
224227

225228
add_iceberg_test(scan_test
226229
USE_BUNDLE

0 commit comments

Comments
 (0)