diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index 639d48837b56..1a91e1bdf2f5 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index ae4b8554cd05..86d2ef6c465d 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/docs-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index 592de5a211fe..046d346f4af2 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/docs-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 850bd4b96144..bfe0aaa45460 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -35,7 +35,7 @@ on: - '.github/workflows/docs-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml index c1318fb4b908..3506e41208cb 100644 --- a/.github/workflows/kafka-connect-ci.yml +++ b/.github/workflows/kafka-connect-ci.yml @@ -36,7 +36,7 @@ on: - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/labeler.yml' - '.github/workflows/license-check.yml' - '.github/workflows/open-api.yml' diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index e9b06b9ffbaf..b97d6ffe4351 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -53,4 +53,8 @@ jobs: run: | ./gradlew printVersion ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW" - ./gradlew -DflinkVersions= -DsparkVersions=3.5,4.0 -DscalaVersion=2.13 -DkafkaVersions=3 publishApachePublicationToMavenRepository -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW" + ./gradlew -DflinkVersions= -DsparkVersions=3.5 -DscalaVersion=2.13 -DkafkaVersions= \ + :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository \ + :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository \ + :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository \ + -PmavenUser="$NEXUS_USER" -PmavenPassword="$NEXUS_PW" diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index f23ad334789f..08642d6f435c 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -36,7 +36,7 @@ on: - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' + - '.github/workflows/jmh-benchmarks.yml' - '.github/workflows/kafka-connect-ci.yml' - '.github/workflows/cve-scan.yml' - '.github/workflows/labeler.yml' @@ -81,6 +81,7 @@ jobs: # Keep matrix <= 20 jobs; exceeding it queues a second wave and slows CI. max-parallel: 20 matrix: + # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) jvm: [17, 21] event_name: ['${{ github.event_name }}'] spark: ['3.5', '4.0', '4.1', '4.2'] @@ -89,8 +90,7 @@ jobs: # so they run as concurrent jobs rather than serially in one job. tests: [core, extensions] exclude: - # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) - # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831) + # Spark 4.0+ only supports Scala 2.13. - spark: '4.0' scala: '2.12' - spark: '4.1' diff --git a/build.gradle b/build.gradle index f98d46bc59d0..38bdb467c6b8 100644 --- a/build.gradle +++ b/build.gradle @@ -1258,8 +1258,11 @@ project(':iceberg-bom') { // only contain artifacts for that single Scala version. The following code ensures that // the BOM references the artifacts for all Scala versions. def sparkScalaPattern = ~"(.*)-([0-9][.][0-9]+)_([0-9][.][0-9]+)" + // Only include Spark versions published in binary releases. def sparkScalaVersions = [ "3.5": ["2.12", "2.13"], + "4.0": ["2.13"], + "4.1": ["2.13"], ] rootProject.allprojects.forEach { // Do not include ':iceberg-spark', the bom itself and the root project.