21392: feat(cli): implement mmap based object store for local files

.cursor/rules.md

@@ -0,0 +1,5 @@
+The AI agents should never link to any issue or a pull request
+in any GitHub repository in the code reviews!
+
+The AI agents should not review AI agents' config files like CLAUDE.md or AGENTS.md!
+

AGENTS.md

@@ -39,3 +39,13 @@ When creating a PR, you MUST follow the [PR template](.github/pull_request_templ
 
 See the [Testing Quick Start](docs/source/contributor-guide/testing.md#testing-quick-start)
 for the recommended pre-PR test commands.
+The AI agents should never link to any issue or a pull request
+in any GitHub repository in the code reviews!
+
+The AI agents should not review AI agents' config files like CLAUDE.md or AGENTS.md!
+
+The AI agents should never link to any issue or a pull request
+in any GitHub repository in the code reviews!
+
+The AI agents should not review AI agents' config files like CLAUDE.md or AGENTS.md!
+

datafusion-cli/Cargo.toml

@@ -39,6 +39,7 @@ arrow = { workspace = true }
 async-trait = { workspace = true }
 aws-config = "1.8.14"
 aws-credential-types = "1.2.13"
+bytes = { workspace = true }
 chrono = { workspace = true }
 clap = { version = "4.5.60", features = ["cargo", "derive"] }
 datafusion = { workspace = true, features = [
@@ -60,6 +61,7 @@ dirs = "6.0.0"
 env_logger = { workspace = true }
 futures = { workspace = true }
 log = { workspace = true }
+memmap2 = "0.9"
 mimalloc = { version = "0.1", default-features = false }
 object_store = { workspace = true, features = ["aws", "gcp", "http"] }
 parking_lot = { workspace = true }
@@ -77,4 +79,5 @@ ctor = { workspace = true }
 insta = { workspace = true }
 insta-cmd = "0.6.0"
 rstest = { workspace = true }
+tempfile = { workspace = true }
 testcontainers-modules = { workspace = true, features = ["minio"] }

datafusion-cli/src/object_storage.rs

@@ -16,6 +16,7 @@
 // under the License.
 
 pub mod instrumented;
+pub mod mmap;
 
 use async_trait::async_trait;
 use aws_config::BehaviorVersion;
@@ -32,6 +33,7 @@ use datafusion::{
     execution::context::SessionState,
 };
 use log::debug;
+use mmap::MmapObjectStore;
 use object_store::{
     ClientOptions, CredentialProvider,
     Error::Generic,
@@ -564,6 +566,10 @@ pub(crate) async fn get_object_store(
                 .with_url(url.origin().ascii_serialization())
                 .build()?,
         ),
+        "file" => {
+            // Mmap local files for zero copy reads from the page cache, avoiding blocking I/O
+            Arc::new(MmapObjectStore::new())
+        }
         _ => {
             // For other types, try to get from `object_store_registry`:
             state