Merge branch 'apache:main' into copy_to_preserve_ordering

Author: bert-beyondloops

.github/workflows/rust.yml

@@ -218,6 +218,8 @@ jobs:
         run: cargo check --profile ci --no-default-features -p datafusion --features=string_expressions
       - name: Check datafusion (unicode_expressions)
         run: cargo check --profile ci --no-default-features -p datafusion --features=unicode_expressions
+      - name: Check parquet encryption (parquet_encryption)
+        run: cargo check --profile ci --no-default-features -p datafusion --features=parquet_encryption
 
   # Check datafusion-functions crate features
   #
@@ -312,18 +314,6 @@ jobs:
           fetch-depth: 1
       - name: Setup Rust toolchain
         run: rustup toolchain install stable
-      - name: Setup Minio - S3-compatible storage
-        run: |
-          docker run -d --name minio-container \
-            -p 9000:9000 \
-            -e MINIO_ROOT_USER=TEST-DataFusionLogin -e MINIO_ROOT_PASSWORD=TEST-DataFusionPassword \
-            -v $(pwd)/datafusion/core/tests/data:/source quay.io/minio/minio \
-            server /data
-          docker exec minio-container /bin/sh -c "\
-            mc ready local
-            mc alias set localminio http://localhost:9000 TEST-DataFusionLogin TEST-DataFusionPassword && \
-            mc mb localminio/data && \
-            mc cp -r /source/* localminio/data"
       - name: Run tests (excluding doctests)
         env:
           RUST_BACKTRACE: 1
@@ -335,9 +325,6 @@ jobs:
         run: cargo test --profile ci -p datafusion-cli  --lib --tests --bins
       - name: Verify Working Directory Clean
         run: git diff --exit-code
-      - name: Minio Output
-        if: ${{ !cancelled() }}
-        run: docker logs minio-container
 
 
   linux-test-example:

Cargo.lock

@@ -150,6 +150,7 @@ env_logger = "0.11"
 futures = "0.3"
 half = { version = "2.6.0", default-features = false }
 hashbrown = { version = "0.14.5", features = ["raw"] }
+hex = { version = "0.4.3" }
 indexmap = "2.10.0"
 itertools = "0.14"
 log = "^0.4"
@@ -173,6 +174,8 @@ rstest = "0.25.0"
 serde_json = "1"
 sqlparser = { version = "0.55.0", default-features = false, features = ["std", "visitor"] }
 tempfile = "3"
+testcontainers = { version = "0.24", features = ["default"] }
+testcontainers-modules = { version = "0.12" }
 tokio = { version = "1.46", features = ["macros", "rt", "sync"] }
 url = "2.5.4"
 

Cargo.toml

@@ -120,6 +120,7 @@ Default features:
 - `datetime_expressions`: date and time functions such as `to_timestamp`
 - `encoding_expressions`: `encode` and `decode` functions
 - `parquet`: support for reading the [Apache Parquet] format
+- `parquet_encryption`: support for using [Parquet Modular Encryption]
 - `regex_expressions`: regular expression functions, such as `regexp_match`
 - `unicode_expressions`: Include unicode aware functions such as `character_length`
 - `unparser`: enables support to reverse LogicalPlans back into SQL
@@ -134,6 +135,7 @@ Optional features:
 
 [apache avro]: https://avro.apache.org/
 [apache parquet]: https://parquet.apache.org/
+[parquet modular encryption]: https://parquet.apache.org/docs/file-format/data-pages/encryption/
 
 ## DataFusion API Evolution and Deprecation Guidelines
 

README.md

@@ -29,47 +29,26 @@ cargo test
 
 ## Running Storage Integration Tests
 
-By default, storage integration tests are not run. To run them you will need to set `TEST_STORAGE_INTEGRATION=1` and
-then provide the necessary configuration for that object store.
+By default, storage integration tests are not run. These test use the `testcontainers` crate to start up a local MinIO server using docker on port 9000.
 
-For some of the tests, [snapshots](https://datafusion.apache.org/contributor-guide/testing.html#snapshot-testing) are used.
-
-### AWS
-
-To test the S3 integration against [Minio](https://github.com/minio/minio)
-
-First start up a container with Minio and load test files.
+To run them you will need to set `TEST_STORAGE_INTEGRATION`:
 
 ```shell
-docker run -d \
-  --name datafusion-test-minio \
-  -p 9000:9000 \
-  -e MINIO_ROOT_USER=TEST-DataFusionLogin \
-  -e MINIO_ROOT_PASSWORD=TEST-DataFusionPassword \
-  -v $(pwd)/../datafusion/core/tests/data:/source \
-  quay.io/minio/minio server /data
-
-docker exec datafusion-test-minio /bin/sh -c "\
-  mc ready local
-  mc alias set localminio http://localhost:9000 TEST-DataFusionLogin TEST-DataFusionPassword && \
-  mc mb localminio/data && \
-  mc cp -r /source/* localminio/data"
+TEST_STORAGE_INTEGRATION=1 cargo test
 ```
 
-Setup environment
+For some of the tests, [snapshots](https://datafusion.apache.org/contributor-guide/testing.html#snapshot-testing) are used.
 
-```shell
-export TEST_STORAGE_INTEGRATION=1
-export AWS_ACCESS_KEY_ID=TEST-DataFusionLogin
-export AWS_SECRET_ACCESS_KEY=TEST-DataFusionPassword
-export AWS_ENDPOINT=http://127.0.0.1:9000
-export AWS_ALLOW_HTTP=true
-```
+### AWS
 
-Note that `AWS_ENDPOINT` is set without slash at the end.
+S3 integration is tested against [Minio](https://github.com/minio/minio) with [TestContainers](https://github.com/testcontainers/testcontainers-rs)
+This requires Docker to be running on your machine and port 9000 to be free.
 
-Run tests
+If you see an error mentioning "failed to load IMDS session token" such as
 
-```shell
-cargo test
-```
+> ---- object_storage::tests::s3_object_store_builder_resolves_region_when_none_provided stdout ----
+> Error: ObjectStore(Generic { store: "S3", source: "Error getting credentials from provider: an error occurred while loading credentials: failed to load IMDS session token" })
+
+You my need to disable trying to fetch S3 credentials from the environment using the `AWS_EC2_METADATA_DISABLED`, for example:
+
+> $ AWS_EC2_METADATA_DISABLED=true TEST_STORAGE_INTEGRATION=1 cargo test

datafusion-cli/CONTRIBUTING.md

@@ -72,3 +72,5 @@ insta = { workspace = true }
 insta-cmd = "0.6.0"
 predicates = "3.0"
 rstest = { workspace = true }
+testcontainers = { workspace = true }
+testcontainers-modules = { workspace = true, features = ["minio"] }