512: feat!: Support enums and tuples in SchemaAwareSerializer and implement SchemaAwareDeserializer

CLAUDE.md

@@ -0,0 +1 @@
+Never link to any issue or a pull request in any Github repository

avro/examples/test_interop_single_object_encoding.rs

@@ -74,7 +74,9 @@ fn test_write(expected: &[u8]) {
 
 fn test_read(encoded: Vec<u8>) {
     let mut encoded = &encoded[..];
-    let read_message = apache_avro::GenericSingleObjectReader::new(InteropMessage::get_schema())
+    let read_message = apache_avro::GenericSingleObjectReader::builder()
+        .schema(InteropMessage::get_schema())
+        .build()
         .expect("Resolving failed")
         .read_value(&mut encoded)
         .expect("Decoding failed");

avro/src/bigdecimal.rs

@@ -69,18 +69,23 @@ pub(crate) fn deserialize_big_decimal(mut bytes: &[u8]) -> AvroResult<BigDecimal
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::{Codec, Reader, Schema, Writer, error::Error, from_value, types::Record};
-    use apache_avro_test_helper::TestResult;
-    use bigdecimal::{One, Zero};
-    use pretty_assertions::assert_eq;
     use std::{
         fs::File,
         io::BufReader,
         ops::{Div, Mul},
         str::FromStr,
     };
 
+    use apache_avro_test_helper::TestResult;
+    use bigdecimal::{One, Zero};
+    use pretty_assertions::assert_eq;
+
+    use super::*;
+    use crate::{
+        Codec, Reader, Schema, Writer, error::Error, reader::datum::GenericDatumReader,
+        types::Record, writer::datum::GenericDatumWriter,
+    };
+
     #[test]
     fn test_avro_3779_bigdecimal_serial() -> TestResult {
         let value: BigDecimal =
@@ -190,6 +195,43 @@ mod tests {
             big_decimal: BigDecimal,
         }
 
+        let schema_str = r#"
+        {
+          "type": "record",
+          "name": "Test",
+          "fields": [
+            {
+              "name": "big_decimal",
+              "type": "string"
+            }
+          ]
+        }
+        "#;
+        let schema = Schema::parse_str(schema_str)?;
+
+        let test = Test::default();
+
+        let serialized = GenericDatumWriter::builder(&schema)
+            .build()?
+            .write_ser_to_vec(&test)?;
+        let value: Test = GenericDatumReader::builder(&schema)
+            .build()?
+            .read_deser(&mut &serialized[..])?;
+
+        assert_eq!(value, test);
+
+        Ok(())
+    }
+
+    #[test]
+    fn avro_rs_338_deserialize_serde_way_with_bigdecimal() -> TestResult {
+        #[derive(Clone, PartialEq, Eq, Debug, Default, serde::Deserialize, serde::Serialize)]
+        #[serde(rename = "test")]
+        struct Test {
+            #[serde(with = "crate::serde::bigdecimal")]
+            big_decimal: BigDecimal,
+        }
+
         let schema_str = r#"
         {
           "type": "record",
@@ -216,11 +258,11 @@ mod tests {
         let wrote_data = writer.into_inner()?;
 
         // read record
-        let mut reader = Reader::new(&wrote_data[..])?;
+        let mut reader = Reader::new(&wrote_data[..])?.into_deser_iter();
 
         let value = reader.next().unwrap()?;
 
-        assert_eq!(test, from_value::<Test>(&value)?);
+        assert_eq!(test, value);
 
         Ok(())
     }

avro/src/documentation/avro_data_model_to_serde.rs

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! # Mapping the Avro data model to the Serde data model
+//!
+//! When manually mapping an Avro schema to Rust types it is important to understand
+//! how the different data models are mapped. When mapping from Rust types to an Avro schema,
+//! see [the documentation for the reverse](super::serde_data_model_to_avro).
+//!
+//! Only the mapping as defined here is supported. Any other behavior might change in a minor version.
+//!
+//! ## Primitive Types
+//! - `null`: `()`
+//! - `boolean`: [`bool`]
+//! - `int`: [`i32`] (or [`i16`], [`i8`], [`u16`], [`u8`])
+//! - `long`: [`i64`] (or [`u32`])
+//! - `float`: [`f32`]
+//! - `double`: [`f64`]
+//! - `bytes`: [`Vec<u8>`](std::vec::Vec) (or any type that uses [`Serializer::serialize_bytes`](serde::Serializer), [`Deserializer::deserialize_bytes`](serde::Deserializer), [`Deserializer::deserialize_byte_buf`](serde::Deserializer))
+//!     - It is required to use [`apache_avro::serde::bytes`] as otherwise Serde will (de)serialize a `Vec` as an array of integers instead.
+//! - `string`: [`String`] (or any type that uses [`Serializer::serialize_str`](serde::Serializer), [`Deserializer::deserialize_str`](serde::Deserializer), [`Deserializer::deserialize_string`](serde::Deserializer))
+//!
+//! ## Complex Types
+//! - `records`: A struct with the same name and fields or a tuple with the same fields.
+//!     - Extra fields can be added to the struct if they are marked with `#[serde(skip)]`
+//! - `enums`: A enum with the same name and unit variants for every symbol
+//!     - The index of the symbol must match the index of the variant
+//! - `arrays`: [`Vec`] (or any type that uses [`Serializer::serialize_seq`](serde::Serializer), [`Deserializer::deserialize_seq`](serde::Deserializer))
+//!     - `[T; N]` is (de)serialized as a tuple by Serde, to (de)serialize them as an `array` use [`apache_avro::serde::array`]
+//! - `maps`: [`HashMap<String, _>`](std::collections::HashMap) (or any type that uses [`Serializer::serialize_map`](serde::Serializer), [`Deserializer::deserialize_map`](serde::Deserializer))
+//! - `unions`: An enum with a variant for each union variant
+//!     - The index of the union variant must match the enum variant
+//!     - A `null` can be a unit variant or a newtype variant with a unit type
+//!     - All other variants must be newtype variants, struct variants, or tuple variants.
+//! - `fixed`: [`Vec<u8>`](std::vec::Vec) (or any type that uses [`Serializer::serialize_bytes`](serde::Serializer), [`Deserializer::deserialize_bytes`](serde::Deserializer), [`Deserializer::deserialize_byte_buf`](serde::Deserializer))
+//!     - It is required to use [`apache_avro::serde::fixed`] as otherwise Serde will (de)serialize a `Vec` as an array of integers instead.
+//!
+//! [`apache_avro::serde::array`]: crate::serde::array
+//! [`apache_avro::serde::bytes`]: crate::serde::bytes
+//! [`apache_avro::serde::fixed`]: crate::serde::fixed

avro/src/documentation/mod.rs

@@ -20,5 +20,7 @@
 //! This module does not contain any code, and is only available during `rustdoc` builds.
 //!
 
+pub mod avro_data_model_to_serde;
 pub mod dynamic;
 pub mod primer;
+pub mod serde_data_model_to_avro;

avro/src/documentation/serde_data_model_to_avro.rs

@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! # Mapping the Serde data model to the Avro data model
+//!
+//! When manually mapping Rust types to an Avro schema, or the reverse, it is important to understand
+//! how the different data models are mapped. When mapping from an Avro schema to Rust types,
+//! see [the documentation for the reverse](super::avro_data_model_to_serde).
+//!
+//! Only the schemas generated by the [`AvroSchema`] derive and the mapping as defined here are
+//! supported. Any other behavior might change in a minor version.
+//!
+//! The following list is based on [the data model defined by Serde](https://serde.rs/data-model.html):
+//! - **14 primitive types**
+//!     - `bool` => [`Schema::Boolean`]
+//!     - `i8`, `i16`, `i32`, `u8`, `u16` => [`Schema::Int`]
+//!     - `i64`, `u32` => [`Schema::Long`]
+//!     - `u64` => [`Schema::Fixed`]`(name: "u64", size: 8)`
+//!         - This is not a `Schema::Long` as that is a signed number of maximum 64 bits.
+//!     - `i128` => [`Schema::Fixed`]`(name: "i128", size: 16)`
+//!     - `u128` => [`Schema::Fixed`]`(name: "u128", size: 16)`
+//!     - `f32` => [`Schema::Float`]
+//!     - `f64` => [`Schema::Double`]
+//!     - `char` => [`Schema::String`]
+//!         - Only one character allowed, deserializer will return an error for strings with more than one character.
+//! - **string** => [`Schema::String`]
+//! - **byte array** => [`Schema::Bytes`] or [`Schema::Fixed`]
+//! - **option** => [`Schema::Union([Schema::Null, _])`](crate::schema::Schema::Union)
+//! - **unit** => [`Schema::Null`]
+//! - **unit struct** => [`Schema::Record`] with the unqualified name equal to the name of the struct and zero fields
+//! - **unit variant** => See [Enums](#enums)
+//! - **newtype struct** => [`Schema::Record`] with the unqualified name equal to the name of the struct and one field
+//! - **newtype variant** => See [Enums](#enums)
+//! - **seq** => [`Schema::Array`]
+//! - **tuple**
+//!     - For tuples with one element, the schema will be the schema of the only element
+//!     - For tuples with more than one element, the schema will be a [`Schema::Record`] with as many fields as there are elements.
+//!       The schema must have the attribute `org.apache.avro.rust.tuple` with the value set to `true`.
+//!     - **Note:** Serde (de)serializes arrays (`[T; N]`) as tuples. To (de)serialize an array as a
+//!       [`Schema::Array`] use [`apache_avro::serde::array`].
+//! - **tuple_struct** => [`Schema::Record`] with the unqualified name equal to the name of the struct and as many fields as there are elements
+//!     - **Note:** Tuple structs with 0 or 1 elements will also be (de)serialized as a [`Schema::Record`]. This
+//!       is different from normal tuples.
+//! - **tuple_variant** => See [Enums](#enums)
+//! - **map** => [`Schema::Map`]
+//!     - **Note:** the key type of the map will be (de)serialized as a [`Schema::String`]
+//! - **struct** => [`Schema::Record`]
+//! - **struct_variant** => See [Enums](#enums)
+//!
+//! ## Enums
+//!
+//! ### Externally tagged
+//! This is the default enum representation for Serde. It can be mapped in three ways to the Avro data model.
+//! For all three options it is important that the enum index matches the Avro index.
+//! - As a [`Schema::Enum`], this is only possible for enums with only unit variants.
+//! - As a [`Schema::Union`] with a [`Schema::Record`] for every variant:
+//!     - **unit_variant** => [`Schema::Record`] named as the variant and with no fields.
+//!     - **newtype_variant** => [`Schema::Record`] named as the variant and with one field.
+//!       The schema must have the attribute `org.apache.avro.rust.union_of_records` with the value set to `true`.
+//!     - **tuple_variant** => [`Schema::Record`] named as the variant and with as many fields as there are element.
+//!     - **struct_variant** => [`Schema::Record`] named as the variant and with a field for every field of the struct variant.
+//! - As a [`Schema::Union`] without the wrapper [`Schema::Record`], all schemas must be unique:
+//!     - **unit_variant** => [`Schema::Null`].
+//!     - **newtype_variant** => The schema of the inner type.
+//!     - **tuple_variant** => [`Schema::Record`] named as the variant and with as many fields as there are element.
+//!     - **struct_variant** => [`Schema::Record`] named as the variant and with a field for every field of the struct variant.
+//!
+//! ### Internally tagged
+//! This enum representation is used by Serde if the attribute `#[serde(tag = "...")]` is used.
+//! It maps to a [`Schema::Record`]. There must be at least one field that is named as the value of the
+//! `tag` attribute. If a field is not used by all variants it must have a `default` set.
+//!
+//! ### Adjacently tagged
+//! This enum representation is used by Serde if the attributes `#[serde(tag = "...", content = "...")]` are used.
+//! It maps to a [`Schema::Record`] with two fields. One field must be named as the value of the `tag`
+//! attribute and use the [`Schema::Enum`] schema. The other field must be named as the value of the
+//! `content` tag and use the [`Schema::Union`] schema.
+//!
+//! ### Untagged
+//! This enum representation is used by Serde if the attribute `#[serde(untagged)]` is used. It maps
+//! to a [`Schema::Union`] with the following schemas:
+//!   - **unit_variant** => [`Schema::Null`].
+//!   - **newtype_variant** => The schema of the inner type.
+//!   - **tuple_variant** => [`Schema::Record`] named as the variant and with as many fields as there are element.
+//!   - **struct_variant** => [`Schema::Record`] named as the variant and with a field for every field of the struct variant.
+//!
+//! [`AvroSchema`]: crate::AvroSchema
+//! [`Schema::Array`]: crate::schema::Schema::Array
+//! [`Schema::Boolean`]: crate::schema::Schema::Boolean
+//! [`Schema::Bytes`]: crate::schema::Schema::Bytes
+//! [`Schema::Double`]: crate::schema::Schema::Double
+//! [`Schema::Enum`]: crate::schema::Schema::Enum
+//! [`Schema::Fixed`]: crate::schema::Schema::Fixed
+//! [`Schema::Float`]: crate::schema::Schema::Float
+//! [`Schema::Int`]: crate::schema::Schema::Int
+//! [`Schema::Long`]: crate::schema::Schema::Long
+//! [`Schema::Map`]: crate::schema::Schema::Map
+//! [`Schema::Null`]: crate::schema::Schema::Null
+//! [`Schema::Record`]: crate::schema::Schema::Record
+//! [`Schema::String`]: crate::schema::Schema::String
+//! [`Schema::Union`]: crate::schema::Schema::Union
+//! [`apache_avro::serde::array`]: crate::serde::array

avro/src/error.rs

@@ -15,11 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::{error::Error as _, fmt};
+
 use crate::{
-    schema::{Name, Schema, SchemaKind, UnionSchema},
+    schema::{Name, RecordSchema, Schema, SchemaKind, UnionSchema},
     types::{Value, ValueKind},
 };
-use std::{error::Error as _, fmt};
 
 /// Errors encountered by Avro.
 ///
@@ -156,8 +157,9 @@ pub enum Details {
     #[error("Union index {index} out of bounds: {num_variants}")]
     GetUnionVariant { index: i64, num_variants: usize },
 
-    #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")]
-    #[error("Enum symbol index out of bounds: {num_variants}")]
+    #[error(
+        "Enum symbol index out of bounds: got {index} but there are only {num_variants} variants"
+    )]
     EnumSymbolIndex { index: usize, num_variants: usize },
 
     #[error("Enum symbol not found {0}")]
@@ -553,32 +555,51 @@ pub enum Details {
     #[error("Decoded integer out of range for i32: {1}: {0}")]
     ZagI32(#[source] std::num::TryFromIntError, i64),
 
-    #[error("unable to read block")]
+    #[error("Did not read any bytes, block is corrupt")]
     ReadBlock,
 
     #[error("Failed to serialize value into Avro value: {0}")]
     SerializeValue(String),
 
-    #[error("Failed to serialize value of type {value_type} using schema {schema:?}: {value}")]
+    #[error("Failed to serialize value of type `{value_type}` using Schema::{schema:?}: {value}")]
     SerializeValueWithSchema {
         value_type: &'static str,
         value: String,
         schema: Schema,
     },
 
-    #[error("Failed to serialize field '{field_name}' for record {record_schema:?}: {error}")]
+    #[error("{position} is not a valid index for fields in {schema:?}")]
+    SerializeRecordUnknownFieldIndex {
+        position: usize,
+        schema: RecordSchema,
+    },
+
+    #[error("Failed to serialize field '{field_name}' of record {record_schema:?}: {error}")]
     SerializeRecordFieldWithSchema {
         field_name: String,
-        record_schema: Schema,
-        error: Box<Error>,
+        record_schema: RecordSchema,
+        error: String,
     },
 
-    #[error("Missing default for skipped field '{field_name}' for schema {schema:?}")]
-    MissingDefaultForSkippedField { field_name: String, schema: Schema },
+    #[error("Missing default for skipped field '{field_name}' of schema {schema:?}")]
+    MissingDefaultForSkippedField {
+        field_name: String,
+        schema: RecordSchema,
+    },
 
     #[error("Failed to deserialize Avro value into value: {0}")]
     DeserializeValue(String),
 
+    #[error("Failed to deserialize value of type {value_type} using schema {schema:?}: {value}")]
+    DeserializeSchemaAware {
+        value_type: &'static str,
+        value: String,
+        schema: Schema,
+    },
+
+    #[error("Only expected `deserialize_identifier` to be called but `{0}` was called")]
+    DeserializeIdentifier(&'static str),
+
     #[error("Failed to write buffer bytes during flush: {0}")]
     WriteBytes(#[source] std::io::Error),
 
@@ -642,6 +663,14 @@ pub enum Details {
         "The implementation of `SchemaNameValidator` is incorrect! It returned an out-of-bounds index or provided a regex that did not capture a group named `name`"
     )]
     InvalidSchemaNameValidatorImplementation,
+
+    #[error(
+        "Not all tuple fields were serialized, expected to serialize element at position {position} of a {total_elements}-tuple but `SerializeTuple::end()` was called"
+    )]
+    SerializeTupleMissingElements {
+        position: usize,
+        total_elements: usize,
+    },
 }
 
 #[derive(thiserror::Error, PartialEq)]

avro/src/lib.rs

@@ -49,6 +49,10 @@
 //! [`Serialize`]: https://docs.rs/serde/latest/serde/trait.Serialize.html
 //! [`Deserialize`]: https://docs.rs/serde/latest/serde/trait.Deserialize.html
 
+// Enable using `fake_variadic` on docs.rs
+#![cfg_attr(docsrs, feature(rustdoc_internals))]
+#![cfg_attr(docsrs, allow(internal_features))]
+
 mod bigdecimal;
 mod bytes;
 mod codec;