[Variant] Use simdutf8 for UTF-8 validation (#7908)

# Which issue does this PR close?

We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.

- Closes #7902.

# Rationale for this change


# What changes are included in this PR?

- add `simdutf8` dependency for parquet-variant
- add a fn `extract_and_validate_utf8_slice` 

# Are these changes tested?

We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code

If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?

# Are there any user-facing changes?

If there are user-facing changes then we may require documentation to be
updated before approving the PR.

If there are any breaking changes to public APIs, please call them out.

---------

Signed-off-by: codephage2020 <tingwangyan2020@163.com>
Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

Author: codephage2020

Cargo.toml

@@ -108,6 +108,8 @@ parquet-variant-compute = { version = "0.1.0", path = "./parquet-variant-json" }
 
 chrono = { version = "0.4.40", default-features = false, features = ["clock"] }
 
+simdutf8 = { version = "0.1.5", default-features = false }
+
 # release inherited profile keeping debug information and symbols
 # for mem/cpu profiling
 [profile.profiling]

arrow-json/Cargo.toml

@@ -49,7 +49,7 @@ serde_json = { version = "1.0", default-features = false, features = ["std"] }
 chrono = { workspace = true }
 lexical-core = { version = "1.0", default-features = false}
 memchr = "2.7.4"
-simdutf8 = "0.1.5"
+simdutf8 = { workspace = true }
 
 [dev-dependencies]
 flate2 = { version = "1", default-features = false, features = ["rust_backend"] }

parquet-variant/Cargo.toml

@@ -37,6 +37,7 @@ arrow-schema = { workspace = true }
 chrono = { workspace = true }
 indexmap = "2.10.0"
 
+simdutf8 = { workspace = true , optional = true }
 
 [lib]
 name = "parquet_variant"
@@ -51,6 +52,10 @@ rand = { version = "0.9", default-features = false, features = [
     "thread_rng",
 ] }
 
+[features]
+default = ["simdutf8"]
+# Enable SIMD UTF-8 validation
+simdutf8 = ["dep:simdutf8"]
 
 [[bench]]
 name = "variant_builder"

parquet-variant/src/utils.rs

@@ -74,13 +74,28 @@ pub(crate) fn first_byte_from_slice(slice: &[u8]) -> Result<u8, ArrowError> {
         .ok_or_else(|| ArrowError::InvalidArgumentError("Received empty bytes".to_string()))
 }
 
-/// Helper to get a &str from a slice at the given offset and range, or an error if invalid.
+/// Helper to get a &str from a slice at the given offset and range, or an error if it contains invalid UTF-8 data.
+#[inline]
 pub(crate) fn string_from_slice(
     slice: &[u8],
     offset: usize,
     range: Range<usize>,
 ) -> Result<&str, ArrowError> {
-    str::from_utf8(slice_from_slice_at_offset(slice, offset, range)?)
+    let offset_buffer = slice_from_slice_at_offset(slice, offset, range)?;
+
+    //Use simdutf8 by default
+    #[cfg(feature = "simdutf8")]
+    {
+        simdutf8::basic::from_utf8(offset_buffer).map_err(|_| {
+            // Use simdutf8::compat to return details about the decoding error
+            let e = simdutf8::compat::from_utf8(offset_buffer).unwrap_err();
+            ArrowError::InvalidArgumentError(format!("encountered non UTF-8 data: {e}"))
+        })
+    }
+
+    //Use std::str if simdutf8 is not enabled
+    #[cfg(not(feature = "simdutf8"))]
+    str::from_utf8(offset_buffer)
         .map_err(|_| ArrowError::InvalidArgumentError("invalid UTF-8 string".to_string()))
 }
 

parquet/Cargo.toml

@@ -70,7 +70,7 @@ twox-hash = { version = "2.0", default-features = false, features = ["xxhash64"]
 paste = { version = "1.0" }
 half = { version = "2.1", default-features = false, features = ["num-traits"] }
 crc32fast = { version = "1.4.2", optional = true, default-features = false }
-simdutf8 = { version = "0.1.5", optional = true, default-features = false }
+simdutf8 = { workspace = true , optional = true }
 ring = { version = "0.17", default-features = false, features = ["std"], optional = true }
 
 [dev-dependencies]