Skip to content

Add JSON serialization and deserialization support for schemas #183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
iand675 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add JSON serialization and deserialization support for schemas #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
89f9f51
First pass at implementing toJSON support
iand675 Jul 22, 2022
1d07b68
Clean up a few spots where there were incomplete pattern matches
iand675 Jul 23, 2022
7e5dc90
fix handling of ignored fields in deconflicted schemas
iand675 Jul 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ When working with `FromAvro` directly it is important to understand the differen

`Schema` (as in the example above) is just a regular data schema for an Avro type.

`ReadSchema` is a similar type, but it is capable of captuting and resolving differences between "_writer_ schema" and "_reader_ schema". See [Specification](https://avro.apache.org/docs/current/spec.html#Schema+Resolution) to learn more about schema resolution and de-conflicting.
`ReadSchema` is a similar type, but it is capable of capturing and resolving differences between "_writer_ schema" and "_reader_ schema". See [Specification](https://avro.apache.org/docs/current/spec.html#Schema+Resolution) to learn more about schema resolution and de-conflicting.

`FromAvro` class requires `ReaderSchema` because with Avro it is possible to read data with a different schema compared to the schema that was used for writing this data.

Expand Down
49 changes: 43 additions & 6 deletions src/Data/Avro/Deriving.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import qualified Data.Aeson as J
import Data.Avro hiding (decode, encode)
import Data.Avro.Encoding.ToAvro (ToAvro (..))
import Data.Avro.Internal.EncodeRaw (putI)
import Data.Avro.JSON
import Data.Avro.Schema.Schema as S
import Data.ByteString (ByteString)
import qualified Data.ByteString as B
Expand All @@ -56,6 +57,7 @@ import qualified Data.List.NonEmpty as NE
import Data.Map (Map)
import Data.Maybe (fromMaybe)
import Data.Semigroup ((<>))
import Data.String (IsString (..))
import qualified Data.Text as Text
import Data.Time (Day, DiffTime, LocalTime, UTCTime)
import Data.UUID (UUID)
Expand Down Expand Up @@ -366,16 +368,27 @@ newNames base n = sequence [newName (base ++ show i) | i <- [1..n]]
------------------------- ToAvro ------------------------------------------------

genToAvro :: DeriveOptions -> Schema -> Q [Dec]
genToAvro opts s@(S.Enum n _ _ _) =
encodeAvroInstance (mkSchemaValueName (namespaceBehavior opts) n)
genToAvro opts s@(S.Enum n _ _ _) = do
baseInstance <- encodeAvroInstance (mkSchemaValueName (namespaceBehavior opts) n)
jsonInstance <- encodeAvroJsonInstance (mkSchemaValueName (namespaceBehavior opts) n)
pure (baseInstance ++ jsonInstance)
where
encodeAvroInstance sname =
[d| instance ToAvro $(conT $ mkDataTypeName (namespaceBehavior opts) n) where
toAvro = $([| \_ x -> putI (fromEnum x) |])
|]
encodeAvroJsonInstance sname =
[d| instance ToAvroJSON $(conT $ mkDataTypeName (namespaceBehavior opts) n) where
toAvroJSON (S.Enum _ _ _ vs) x = case vs V.!? fromEnum x of
Nothing -> error "toAvroJson: unable to find equivalent enum value in new schema"
Just _ -> toJSON originalName
toAvroJSON _ _ = error "ToAvroJson for enum only works for Enum schema values"
|]

genToAvro opts s@(S.Record n _ _ fs) =
encodeAvroInstance (mkSchemaValueName (namespaceBehavior opts) n)
genToAvro opts s@(S.Record n _ _ fs) = do
baseInstance <- encodeAvroInstance (mkSchemaValueName (namespaceBehavior opts) n)
jsonInstance <- encodeAvroJsonInstance (mkSchemaValueName (namespaceBehavior opts) n)
pure (baseInstance ++ jsonInstance)
where
encodeAvroInstance sname =
[d| instance ToAvro $(conT $ mkDataTypeName (namespaceBehavior opts) n) where
Expand All @@ -390,9 +403,26 @@ genToAvro opts s@(S.Record n _ _ fs) =
in listE $ build <$> zip fs names
)
|]
encodeAvroJsonInstance sname =
[d| instance ToAvroJSON $(conT $ mkDataTypeName (namespaceBehavior opts) n) where
toAvroJSON = $(encodeAvroJsonFieldsExp sname)
|]
encodeAvroJsonFieldsExp sname = do
names <- newNames "p_" (length fs)
wn <- varP <$> newName "_"
let con = conP (mkDataTypeName (namespaceBehavior opts) n) (varP <$> names)
lamE [wn, con]
[| J.object
$( let build (fld, n) = [| fromString $(stringE $ Text.unpack $ fldName fld) J..= toAvroJSON (fldType fld) $(varE n) |]
in listE $ build <$> zip fs names
)
|]


genToAvro opts s@(S.Fixed n _ _ _) =
encodeAvroInstance (mkSchemaValueName (namespaceBehavior opts) n)
genToAvro opts s@(S.Fixed n _ _ _) = do
baseInstance <- encodeAvroInstance (mkSchemaValueName (namespaceBehavior opts) n)
jsonInstance <- encodeAvroJsonInstance (mkSchemaValueName (namespaceBehavior opts) n)
pure (baseInstance ++ jsonInstance)
where
encodeAvroInstance sname =
[d| instance ToAvro $(conT $ mkDataTypeName (namespaceBehavior opts) n) where
Expand All @@ -401,6 +431,13 @@ genToAvro opts s@(S.Fixed n _ _ _) =
wc <- newName "_"
lamE [varP wc, conP (mkDataTypeName (namespaceBehavior opts) n) [varP x]] [| toAvro $(varE sname) $(varE x) |])
|]
encodeAvroJsonInstance sname =
[d| instance ToAvroJSON $(conT $ mkDataTypeName (namespaceBehavior opts) n) where
toAvroJSON = $(do
x <- newName "x"
wc <- newName "_"
lamE [varP wc, conP (mkDataTypeName (namespaceBehavior opts) n) [varP x]] [| toAvroJSON $(varE sname) $(varE x) |])
|]
genToAvro _ _ = pure []

schemaDef :: Name -> Schema -> Q [Dec]
Expand Down
8 changes: 8 additions & 0 deletions src/Data/Avro/EitherN.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,13 +443,15 @@ instance (FromAvro a, FromAvro b, FromAvro c) => FromAvro (Either3 a b c) where
fromAvro (AV.Union _ 1 b) = E3_2 <$> fromAvro b
fromAvro (AV.Union _ 2 c) = E3_3 <$> fromAvro c
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either3 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either3 from a non-union"

instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d) => FromAvro (Either4 a b c d) where
fromAvro (AV.Union _ 0 a) = E4_1 <$> fromAvro a
fromAvro (AV.Union _ 1 b) = E4_2 <$> fromAvro b
fromAvro (AV.Union _ 2 c) = E4_3 <$> fromAvro c
fromAvro (AV.Union _ 3 d) = E4_4 <$> fromAvro d
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either4 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either4 from a non-union"

instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e) => FromAvro (Either5 a b c d e) where
fromAvro (AV.Union _ 0 a) = E5_1 <$> fromAvro a
Expand All @@ -458,6 +460,7 @@ instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e) => FromAvr
fromAvro (AV.Union _ 3 d) = E5_4 <$> fromAvro d
fromAvro (AV.Union _ 4 e) = E5_5 <$> fromAvro e
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either5 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either5 from a non-union"

instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f) => FromAvro (Either6 a b c d e f) where
fromAvro (AV.Union _ 0 a) = E6_1 <$> fromAvro a
Expand All @@ -467,6 +470,7 @@ instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f
fromAvro (AV.Union _ 4 e) = E6_5 <$> fromAvro e
fromAvro (AV.Union _ 5 f) = E6_6 <$> fromAvro f
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either6 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either6 from a non-union"

instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f, FromAvro g) => FromAvro (Either7 a b c d e f g) where
fromAvro (AV.Union _ 0 a) = E7_1 <$> fromAvro a
Expand All @@ -477,6 +481,7 @@ instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f
fromAvro (AV.Union _ 5 f) = E7_6 <$> fromAvro f
fromAvro (AV.Union _ 6 g) = E7_7 <$> fromAvro g
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either7 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either7 from a non-union"

instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f, FromAvro g, FromAvro h) => FromAvro (Either8 a b c d e f g h) where
fromAvro (AV.Union _ 0 a) = E8_1 <$> fromAvro a
Expand All @@ -488,6 +493,7 @@ instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f
fromAvro (AV.Union _ 6 g) = E8_7 <$> fromAvro g
fromAvro (AV.Union _ 7 h) = E8_8 <$> fromAvro h
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either8 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either8 from a non-union"

instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f, FromAvro g, FromAvro h, FromAvro i) => FromAvro (Either9 a b c d e f g h i) where
fromAvro (AV.Union _ 0 a) = E9_1 <$> fromAvro a
Expand All @@ -500,6 +506,7 @@ instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f
fromAvro (AV.Union _ 7 h) = E9_8 <$> fromAvro h
fromAvro (AV.Union _ 8 i) = E9_9 <$> fromAvro i
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either9 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either9 from a non-union"

instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f, FromAvro g, FromAvro h, FromAvro i, FromAvro j) => FromAvro (Either10 a b c d e f g h i j) where
fromAvro (AV.Union _ 0 a) = E10_1 <$> fromAvro a
Expand All @@ -513,6 +520,7 @@ instance (FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e, FromAvro f
fromAvro (AV.Union _ 8 i) = E10_9 <$> fromAvro i
fromAvro (AV.Union _ 9 j) = E10_10 <$> fromAvro j
fromAvro (AV.Union _ n _) = Left ("Unable to decode Either10 from a position #" <> show n)
fromAvro _ = Left "Unable to decode Either10 from a non-union"

putIndexedValue :: ToAvro a => Int -> V.Vector Schema -> a -> Builder
putIndexedValue i opts x = putI i <> toAvro (V.unsafeIndex opts i) x
Expand Down
1 change: 1 addition & 0 deletions src/Data/Avro/Encoding/FromAvro.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ module Data.Avro.Encoding.FromAvro
-- ** For internal use
, Value(..)
, getValue
, convertValue
)
where

Expand Down
Loading