Merge branch 'main' into file-stats-cache

Author: mkleen

Cargo.lock

@@ -36,15 +36,16 @@ ROOT_DIR="$(git rev-parse --show-toplevel)"
 EXAMPLES_DIR="$ROOT_DIR/datafusion-examples"
 README="$EXAMPLES_DIR/README.md"
 README_NEW="$EXAMPLES_DIR/README-NEW.md"
+PRETTIER_VERSION="2.7.1"
 
 echo "▶ Generating examples README (Rust generator)…"
 cargo run --quiet \
   --manifest-path "$EXAMPLES_DIR/Cargo.toml" \
   --bin examples-docs \
   > "$README_NEW"
 
-echo "▶ Formatting generated README with Prettier…"
-npx prettier@2.7.1 \
+echo "▶ Formatting generated README with prettier ${PRETTIER_VERSION}…"
+npx "prettier@${PRETTIER_VERSION}" \
   --parser markdown \
   --write "$README_NEW"
 
@@ -60,7 +61,7 @@ if ! diff -u "$README" "$README_NEW" > /tmp/examples-readme.diff; then
   echo "To update the README locally, run:"
   echo ""
   echo "  cargo run --bin examples-docs \\"
-  echo "    | npx prettier@2.7.1 --parser markdown --write \\"
+  echo "    | npx prettier@${PRETTIER_VERSION} --parser markdown --write \\"
   echo "    > datafusion-examples/README.md"
   echo ""
   echo "Diff:"

ci/scripts/check_examples_docs.sh

@@ -300,7 +300,7 @@ impl StatementExecutor {
                 let curr_num_rows = batch.num_rows();
                 // Stop collecting results if the number of rows exceeds the limit
                 // results batch should include the last batch that exceeds the limit
-                if row_count < max_rows + curr_num_rows {
+                if row_count < max_rows.saturating_add(curr_num_rows) {
                     // Try to grow the reservation to accommodate the batch in memory
                     reservation.try_grow(get_record_batch_memory_size(&batch))?;
                     results.push(batch);

datafusion-cli/src/exec.rs

@@ -40,6 +40,7 @@ arrow = { workspace = true }
 arrow-schema = { workspace = true }
 datafusion = { workspace = true, default-features = true, features = ["parquet_encryption"] }
 datafusion-common = { workspace = true }
+nom = "8.0.0"
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }
 

datafusion-examples/Cargo.toml

@@ -34,7 +34,9 @@
 //! cargo run --bin examples-docs -- dataframe
 //! ```
 
-use datafusion_examples::utils::examples_docs::{RepoLayout, generate_examples_readme};
+use datafusion_examples::utils::example_metadata::{
+    RepoLayout, generate_examples_readme,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let layout = RepoLayout::detect()?;

datafusion-examples/src/bin/examples-docs.rs

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Utilities for discovering example groups in the repository filesystem.
+//!
+//! An example group is defined as a directory containing a `main.rs` file
+//! under the examples root. This module is intentionally filesystem-focused
+//! and does not perform any parsing or rendering.
+
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use datafusion::error::Result;
+
+/// Discovers all example group directories under the given root.
+///
+/// A directory is considered an example group if it contains a `main.rs` file.
+pub fn discover_example_groups(root: &Path) -> Result<Vec<PathBuf>> {
+    let mut groups = Vec::new();
+    for entry in fs::read_dir(root)? {
+        let entry = entry?;
+        let path = entry.path();
+
+        if path.is_dir() && path.join("main.rs").exists() {
+            groups.push(path);
+        }
+    }
+    groups.sort();
+    Ok(groups)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::fs::{self, File};
+
+    use tempfile::TempDir;
+
+    #[test]
+    fn discover_example_groups_finds_dirs_with_main_rs() -> Result<()> {
+        let tmp = TempDir::new()?;
+        let root = tmp.path();
+
+        // valid example group
+        let group1 = root.join("group1");
+        fs::create_dir(&group1)?;
+        File::create(group1.join("main.rs"))?;
+
+        // not an example group
+        let group2 = root.join("group2");
+        fs::create_dir(&group2)?;
+
+        let groups = discover_example_groups(root)?;
+
+        assert_eq!(groups.len(), 1);
+        assert_eq!(groups[0], group1);
+
+        Ok(())
+    }
+}

datafusion-examples/src/utils/example_metadata/discover.rs

@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Repository layout utilities.
+//!
+//! This module provides a small helper (`RepoLayout`) that encapsulates
+//! knowledge about the DataFusion repository structure, in particular
+//! where example groups are located relative to the repository root.
+
+use std::path::{Path, PathBuf};
+
+use datafusion::error::{DataFusionError, Result};
+
+/// Describes the layout of a DataFusion repository.
+///
+/// This type centralizes knowledge about where example-related
+/// directories live relative to the repository root.
+#[derive(Debug, Clone)]
+pub struct RepoLayout {
+    root: PathBuf,
+}
+
+impl From<&Path> for RepoLayout {
+    fn from(path: &Path) -> Self {
+        Self {
+            root: path.to_path_buf(),
+        }
+    }
+}
+
+impl RepoLayout {
+    /// Creates a layout from an explicit repository root.
+    pub fn from_root(root: PathBuf) -> Self {
+        Self { root }
+    }
+
+    /// Detects the repository root based on `CARGO_MANIFEST_DIR`.
+    ///
+    /// This is intended for use from binaries inside the workspace.
+    pub fn detect() -> Result<Self> {
+        let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+
+        let root = manifest_dir.parent().ok_or_else(|| {
+            DataFusionError::Execution(
+                "CARGO_MANIFEST_DIR does not have a parent".to_string(),
+            )
+        })?;
+
+        Ok(Self {
+            root: root.to_path_buf(),
+        })
+    }
+
+    /// Returns the repository root directory.
+    pub fn root(&self) -> &Path {
+        &self.root
+    }
+
+    /// Returns the `datafusion-examples/examples` directory.
+    pub fn examples_root(&self) -> PathBuf {
+        self.root.join("datafusion-examples").join("examples")
+    }
+
+    /// Returns the directory for a single example group.
+    ///
+    /// Example: `examples/udf`
+    pub fn example_group_dir(&self, group: &str) -> PathBuf {
+        self.examples_root().join(group)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn detect_sets_non_empty_root() -> Result<()> {
+        let layout = RepoLayout::detect()?;
+        assert!(!layout.root().as_os_str().is_empty());
+        Ok(())
+    }
+
+    #[test]
+    fn examples_root_is_under_repo_root() -> Result<()> {
+        let layout = RepoLayout::detect()?;
+        let examples_root = layout.examples_root();
+        assert!(examples_root.starts_with(layout.root()));
+        assert!(examples_root.ends_with("datafusion-examples/examples"));
+        Ok(())
+    }
+
+    #[test]
+    fn example_group_dir_appends_group_name() -> Result<()> {
+        let layout = RepoLayout::detect()?;
+        let group_dir = layout.example_group_dir("foo");
+        assert!(group_dir.ends_with("datafusion-examples/examples/foo"));
+        Ok(())
+    }
+}

datafusion-examples/src/utils/example_metadata/layout.rs

@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Documentation generator for DataFusion examples.
+//!
+//! # Design goals
+//!
+//! - Keep README.md in sync with runnable examples
+//! - Fail fast on malformed documentation
+//!
+//! # Overview
+//!
+//! Each example group corresponds to a directory under
+//! `datafusion-examples/examples/<group>` containing a `main.rs` file.
+//! Documentation is extracted from structured `//!` comments in that file.
+//!
+//! For each example group, the generator produces:
+//!
+//! ```text
+//! ## <Group Name> Examples
+//! ### Group: `<group>`
+//! #### Category: Single Process | Distributed
+//!
+//! | Subcommand | File Path | Description |
+//! ```
+//!
+//! # Usage
+//!
+//! Generate documentation for a single group only:
+//!
+//! ```bash
+//! cargo run --bin examples-docs -- dataframe
+//! ```
+//!
+//! Generate documentation for all examples:
+//!
+//! ```bash
+//! cargo run --bin examples-docs  
+//! ```
+
+pub mod discover;
+pub mod layout;
+pub mod model;
+pub mod parser;
+pub mod render;
+
+#[cfg(test)]
+pub mod test_utils;
+
+pub use layout::RepoLayout;
+pub use model::{Category, ExampleEntry, ExampleGroup, GroupName};
+pub use parser::parse_main_rs_docs;
+pub use render::generate_examples_readme;