diff --git a/CHANGES.md b/CHANGES.md index 1bd807abe1e4..bcff85048b65 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -69,6 +69,7 @@ * Add Datadog IO support (Java) ([#37318](https://github.com/apache/beam/issues/37318)). * Remove Pubsublite IO support, since service will be deprecated in March 2026. ([#37375](https://github.com/apache/beam/issues/37375)). +* S3-compatible storage support added for Iceberg managed I/O (Java) ([#37614](https://github.com/apache/beam/issues/37614)). ## New Features / Improvements diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index be9e3704e6c2..8028e8020b69 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -83,6 +83,7 @@ dependencies { runtimeOnly project(":sdks:java:io:iceberg") runtimeOnly project(":sdks:java:io:iceberg:hive") runtimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow") + runtimeOnly project(":sdks:java:io:iceberg:aws") } runtimeOnly library.java.kafka_clients diff --git a/sdks/java/io/iceberg/aws/build.gradle b/sdks/java/io/iceberg/aws/build.gradle new file mode 100644 index 000000000000..f3ae7d1b8041 --- /dev/null +++ b/sdks/java/io/iceberg/aws/build.gradle @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.sdk.io.iceberg.aws', + exportJavadoc: false, + publish: false, // it's an intermediate jar for io-expansion-service +) + +description = "Apache Beam :: SDKs :: Java :: IO :: Iceberg :: AWS" +ext.summary = "Runtime dependencies needed for S3-compatible storage." + +def iceberg_version = "1.10.0" + +dependencies { + // dependencies needed to run with S3-compatible storage + // these dependencies are going to be included in io-expansion-service + runtimeOnly "org.apache.iceberg:iceberg-aws:$iceberg_version" + runtimeOnly "org.apache.iceberg:iceberg-aws-bundle:$iceberg_version" +} diff --git a/settings.gradle.kts b/settings.gradle.kts index a37c57d043c3..9a86be694e59 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -368,6 +368,8 @@ include("sdks:java:io:iceberg:hive") findProject(":sdks:java:io:iceberg:hive")?.name = "hive" include("sdks:java:io:iceberg:bqms") findProject(":sdks:java:io:iceberg:bqms")?.name = "bqms" +include("sdks:java:io:iceberg:aws") +findProject(":sdks:java:io:iceberg:aws")?.name = "aws" include("it:clickhouse") findProject(":it:clickhouse")?.name = "clickhouse" include("sdks:java:extensions:sql:iceberg")