From baa09f2f461c7333c9c94fc3152f86bd776f1563 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Thu, 10 Sep 2026 13:02:26 +0900 Subject: [PATCH 1/3] Build: Fix Spark publication and CI configuration Avoid republishing unrelated snapshot artifacts and preserve complete release BOM constraints. Correct stale benchmark workflow filters and clarify the supported Spark matrix. Generated-by: Codex --- .github/workflows/delta-conversion-ci.yml | 2 +- .github/workflows/flink-ci.yml | 2 +- .github/workflows/hive-ci.yml | 2 +- .github/workflows/java-ci.yml | 2 +- .github/workflows/kafka-connect-ci.yml | 2 +- .github/workflows/publish-snapshot.yml | 6 +++++- .github/workflows/spark-ci.yml | 7 ++++--- build.gradle | 3 +++ dev/stage-binaries.sh | 2 +- 9 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index 639d48837b56..1a91e1bdf2f5 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index ae4b8554cd05..86d2ef6c465d 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/docs-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index 592de5a211fe..046d346f4af2 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/docs-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 850bd4b96144..bfe0aaa45460 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/docs-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml index c1318fb4b908..3506e41208cb 100644 --- a/.github/workflows/kafka-connect-ci.yml +++ b/.github/workflows/kafka-connect-ci.yml @@ -36,7 +36,7 @@ on: - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/labeler.yml' - '.github/workflows/license-check.yml' - '.github/workflows/open-api.yml' diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index e9b06b9ffbaf..b97d6ffe4351 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -53,4 +53,8 @@ jobs: run: | ./gradlew printVersion ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW" - ./gradlew -DflinkVersions= -DsparkVersions=3.5,4.0 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW" + ./gradlew -DflinkVersions= -DsparkVersions=3.5 -DscalaVersion=2.13 -DkafkaVersions= \ + :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository \ + :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository \ + :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository \ + -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW" diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index f23ad334789f..52e28184e454 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -36,7 +36,7 @@ on: - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' @@ -81,6 +81,8 @@ jobs: # Keep matrix <= 20 jobs; exceeding it queues a second wave and slows CI. max-parallel: 20 matrix: + # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) + # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831) jvm: [17, 21] event_name: ['${{ github.event_name }}'] spark: ['3.5', '4.0', '4.1', '4.2'] @@ -89,8 +91,7 @@ jobs: # so they run as concurrent jobs rather than serially in one job. tests: [core, extensions] exclude: - # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) - # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831) + # Spark 4.0+ only supports Scala 2.13. - spark: '4.0' scala: '2.12' - spark: '4.1' diff --git a/build.gradle b/build.gradle index f98d46bc59d0..38bdb467c6b8 100644 --- a/build.gradle +++ b/build.gradle @@ -1258,8 +1258,11 @@ project(':iceberg-bom') { // only contain artifacts for that single Scala version. The following code ensures that // the BOM references the artifacts for all Scala versions. def sparkScalaPattern = ~"(.*)-([0-9][.][0-9]+)_([0-9][.][0-9]+)" + // Only include Spark versions published in binary releases. def sparkScalaVersions = [ "3.5": ["2.12", "2.13"], + "4.0": ["2.13"], + "4.1": ["2.13"], ] rootProject.allprojects.forEach { // Do not include ':iceberg-spark', the bom itself and the root project. diff --git a/dev/stage-binaries.sh b/dev/stage-binaries.sh index bf8576d95fcc..fedbee2f1c87 100755 --- a/dev/stage-binaries.sh +++ b/dev/stage-binaries.sh @@ -20,6 +20,7 @@ SCALA_VERSION=2.12 FLINK_VERSIONS=1.20,2.1,2.2,2.3 +# Spark 4.2 is intentionally excluded because it is not part of binary releases. SPARK_VERSIONS=3.5,4.0,4.1 KAFKA_VERSIONS=3 @@ -29,4 +30,3 @@ KAFKA_VERSIONS=3 # Flink does not yet support 2.13 (and is largely dropping a user-facing dependency on Scala). Hive doesn't need a Scala specification. ./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.5 :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache # Spark 4.0+ only supports Scala 2.13. no need to specify scalaVersion - From 029a0ecacc4fee5965a480a9aa138bea47b8f8df Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Fri, 11 Sep 2026 13:01:16 +0900 Subject: [PATCH 2/3] Build: Remove outdated Spark Java 21 support comment Generated-by: Codex --- .github/workflows/spark-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 52e28184e454..08642d6f435c 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -82,7 +82,6 @@ jobs: max-parallel: 20 matrix: # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) - # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831) jvm: [17, 21] event_name: ['${{ github.event_name }}'] spark: ['3.5', '4.0', '4.1', '4.2'] From aed92f5945feb8cc8bf6790d6cf18d34b94b1299 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Fri, 11 Sep 2026 13:02:44 +0900 Subject: [PATCH 3/3] Build: Restore binary staging script Keep dev/stage-binaries.sh unchanged in this PR. Generated-by: Codex --- dev/stage-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/stage-binaries.sh b/dev/stage-binaries.sh index fedbee2f1c87..bf8576d95fcc 100755 --- a/dev/stage-binaries.sh +++ b/dev/stage-binaries.sh @@ -20,7 +20,6 @@ SCALA_VERSION=2.12 FLINK_VERSIONS=1.20,2.1,2.2,2.3 -# Spark 4.2 is intentionally excluded because it is not part of binary releases. SPARK_VERSIONS=3.5,4.0,4.1 KAFKA_VERSIONS=3 @@ -30,3 +29,4 @@ KAFKA_VERSIONS=3 # Flink does not yet support 2.13 (and is largely dropping a user-facing dependency on Scala). Hive doesn't need a Scala specification. ./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.5 :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository --no-parallel --no-configuration-cache # Spark 4.0+ only supports Scala 2.13. no need to specify scalaVersion +