feat: avro_1_12 feature

Introduce the "avro_1_12" feature flag and use it to guard the
behavior of JSON null defaults for union types having null schema in a
position other than the first.

Author: mzabaluev

arrow-avro/Cargo.toml

@@ -44,6 +44,7 @@ md5 = ["dep:md5"]
 sha256 = ["dep:sha2"]
 small_decimals = []
 avro_custom_types = ["dep:arrow-select"]
+avro_1_12 = []
 
 # Enable async APIs
 async = ["futures", "tokio"]

arrow-avro/src/codec.rs

@@ -251,6 +251,52 @@ impl AvroDataType {
         self.nullability
     }
 
+    // Returns `Ok` if this data type accepts a JSON null default value,
+    // according to Avro schema rules prior to spec version 1.12, otherwise
+    // returns an `Err` with a schema error.
+    // Prior to 1.12, Avro only allowed default values matching the first branch of a union.
+    #[cfg(not(feature = "avro_1_12"))]
+    fn validate_null_default(&self) -> Result<(), ArrowError> {
+        match self.codec() {
+            Codec::Null => Ok(()),
+            Codec::Union(encodings, _, _)
+                if encodings
+                    .first()
+                    .map_or(false, |enc| matches!(enc.codec(), Codec::Null)) =>
+            {
+                Ok(())
+            }
+            _ if self.nullability() == Some(Nullability::NullFirst) => Ok(()),
+            _ => Err(ArrowError::SchemaError(
+                    "JSON null default is only valid for `null` type or for a union whose first branch is `null`"
+                        .to_string(),
+                )),
+        }
+    }
+
+    // Returns `Ok` if this data type accepts a JSON null default value,
+    // according to Avro schema rules for spec version 1.12 and later, otherwise
+    // returns an `Err` with a schema error.
+    // Since 1.12, Avro allows default values matching any branch of a union.
+    #[cfg(feature = "avro_1_12")]
+    fn validate_null_default(&self) -> Result<(), ArrowError> {
+        match self.codec() {
+            Codec::Null => Ok(()),
+            Codec::Union(encodings, _, _)
+                if encodings
+                    .iter()
+                    .any(|enc| matches!(enc.codec(), Codec::Null)) =>
+            {
+                Ok(())
+            }
+            _ if self.nullability().is_some() => Ok(()),
+            _ => Err(ArrowError::SchemaError(
+                    "JSON null default is only valid for `null` type or for a union with a `null` branch"
+                        .to_string(),
+                )),
+        }
+    }
+
     #[inline]
     fn parse_default_literal(&self, default_json: &Value) -> Result<AvroLiteral, ArrowError> {
         fn expect_string<'v>(
@@ -313,21 +359,10 @@ impl AvroDataType {
             }
         }
 
-        // Handle JSON nulls per-spec: allowed only for `null` type or unions with null
+        // Handle JSON nulls per the spec rules
         if default_json.is_null() {
-            return match self.codec() {
-                Codec::Null => Ok(AvroLiteral::Null),
-                Codec::Union(encodings, _, _)
-                    if encodings.iter().any(|enc| matches!(enc.codec(), Codec::Null)) =>
-                    {
-                        Ok(AvroLiteral::Null)
-                    }
-                _ if self.nullability().is_some() => Ok(AvroLiteral::Null),
-                _ => Err(ArrowError::SchemaError(
-                    "JSON null default is only valid for `null` type or for a union with a `null` branch"
-                        .to_string(),
-                )),
-            };
+            self.validate_null_default()?;
+            return Ok(AvroLiteral::Null);
         }
         let lit = match self.codec() {
             Codec::Null => {
@@ -3282,6 +3317,24 @@ mod tests {
         let lit2 = dt_int_nf.parse_and_store_default(&Value::Null).unwrap();
         assert_eq!(lit2, AvroLiteral::Null);
         assert_default_stored(&dt_int_nf, &Value::Null);
+    }
+
+    #[cfg(not(feature = "avro_1_12"))]
+    #[test]
+    fn test_validate_and_store_default_null_and_nullability_rules_avro_1_11() {
+        let mut dt_int_ns =
+            AvroDataType::new(Codec::Int32, HashMap::new(), Some(Nullability::NullSecond));
+        let err2 = dt_int_ns.parse_and_store_default(&Value::Null).unwrap_err();
+        assert!(
+            err2.to_string()
+                .contains("JSON null default is only valid for `null` type"),
+            "unexpected error: {err2}"
+        );
+    }
+
+    #[cfg(feature = "avro_1_12")]
+    #[test]
+    fn test_validate_and_store_default_null_and_nullability_rules_avro_1_12() {
         let mut dt_int_ns =
             AvroDataType::new(Codec::Int32, HashMap::new(), Some(Nullability::NullSecond));
         let lit3 = dt_int_ns.parse_and_store_default(&Value::Null).unwrap();

arrow-avro/src/reader/mod.rs

@@ -2950,15 +2950,14 @@ mod test {
         },"default":{"x":7}}),
                 serde_json::json!({"name":"d_nullable_null","type":["null","int"],"default":null}),
                 serde_json::json!({"name":"d_nullable_value","type":["int","null"],"default":123}),
-                serde_json::json!({"name":"d_nullable_null_second","type":["int","null"],"default":null}),
             ],
         );
         let actual = read_alltypes_with_reader_schema(path, reader_schema);
         let num_rows = actual.num_rows();
         assert!(num_rows > 0, "skippable_types.avro should contain rows");
         assert_eq!(
             actual.num_columns(),
-            23,
+            22,
             "expected exactly our defaulted fields"
         );
         let mut arrays: Vec<Arc<dyn Array>> = Vec::with_capacity(22);
@@ -3085,6 +3084,32 @@ mod test {
         arrays.push(Arc::new(Int32Array::from_iter_values(std::iter::repeat_n(
             123, num_rows,
         ))));
+        let expected = RecordBatch::try_new(actual.schema(), arrays).unwrap();
+        assert_eq!(
+            actual, expected,
+            "defaults should materialize correctly for all fields"
+        );
+    }
+
+    #[cfg(feature = "avro_1_12")]
+    #[test]
+    fn test_schema_resolution_defaults_cases_supported_by_avro_1_12() {
+        let path = "test/data/skippable_types.avro";
+        let reader_schema = make_reader_schema_with_default_fields(
+            path,
+            vec![
+                serde_json::json!({"name":"d_nullable_null_second","type":["int","null"],"default":null}),
+            ],
+        );
+        let actual = read_alltypes_with_reader_schema(path, reader_schema);
+        let num_rows = actual.num_rows();
+        assert!(num_rows > 0, "skippable_types.avro should contain rows");
+        assert_eq!(
+            actual.num_columns(),
+            1,
+            "expected exactly our defaulted fields"
+        );
+        let mut arrays: Vec<Arc<dyn Array>> = Vec::with_capacity(22);
         arrays.push(Arc::new(Int32Array::from_iter(std::iter::repeat_n(
             None::<i32>,
             num_rows,