gen: Create type safe union types

[petedannemann]

Right now it appears that generated union types simply use any, which is not type safe. This means code needs to be written manually to make this type safe. Ideally there would be some option to generate the boilerplate code to make this type safe.

Example schema with a union type:

{
  "namespace": "com.test.avro",
  "type": "record",
  "name": "TestEvent",
  "doc": "Test event",
  "fields": [
    {
      "name": "exampleunion",
      "type": ["string", "long"]
    }
  ]
}

avrogen -pkg avro -o testevent.go TestEvent.avsc

testevent.go

// Code generated by avro/gen. DO NOT EDIT.
package avro

// Test event.
type TestEvent struct {
	Exampleunion any `avro:"exampleunion"`
}

[nrwiersma]

And what would that type safe code look like, given the breadth of what can be in a union?

[petedannemann]

For reference this is what actgardner/gogen-avro does when generating that schema. I understand this is a complex feature 😅 , just trying to voice a problem we ran into when trying to move to this library

# Install if not already done
go install github.com/actgardner/gogen-avro/v10/cmd/...@latest

gogen-avro . TestEvent.avsc

This generated two files.

union_string_long.go

// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT.
/*
 * SOURCE:
 *     TestEvent.avsc
 */
package avro

import (
	"encoding/json"
	"fmt"
	"io"

	"github.com/actgardner/gogen-avro/v10/compiler"
	"github.com/actgardner/gogen-avro/v10/vm"
	"github.com/actgardner/gogen-avro/v10/vm/types"
)

type UnionStringLongTypeEnum int

const (
	UnionStringLongTypeEnumString UnionStringLongTypeEnum = 0

	UnionStringLongTypeEnumLong UnionStringLongTypeEnum = 1
)

type UnionStringLong struct {
	String    string
	Long      int64
	UnionType UnionStringLongTypeEnum
}

func writeUnionStringLong(r UnionStringLong, w io.Writer) error {

	err := vm.WriteLong(int64(r.UnionType), w)
	if err != nil {
		return err
	}
	switch r.UnionType {
	case UnionStringLongTypeEnumString:
		return vm.WriteString(r.String, w)
	case UnionStringLongTypeEnumLong:
		return vm.WriteLong(r.Long, w)
	}
	return fmt.Errorf("invalid value for UnionStringLong")
}

func NewUnionStringLong() UnionStringLong {
	return UnionStringLong{}
}

func (r UnionStringLong) Serialize(w io.Writer) error {
	return writeUnionStringLong(r, w)
}

func DeserializeUnionStringLong(r io.Reader) (UnionStringLong, error) {
	t := NewUnionStringLong()
	deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema()))
	if err != nil {
		return t, err
	}

	err = vm.Eval(r, deser, &t)

	if err != nil {
		return t, err
	}
	return t, err
}

func DeserializeUnionStringLongFromSchema(r io.Reader, schema string) (UnionStringLong, error) {
	t := NewUnionStringLong()
	deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema()))
	if err != nil {
		return t, err
	}

	err = vm.Eval(r, deser, &t)

	if err != nil {
		return t, err
	}
	return t, err
}

func (r UnionStringLong) Schema() string {
	return "[\"string\",\"long\"]"
}

func (_ UnionStringLong) SetBoolean(v bool)   { panic("Unsupported operation") }
func (_ UnionStringLong) SetInt(v int32)      { panic("Unsupported operation") }
func (_ UnionStringLong) SetFloat(v float32)  { panic("Unsupported operation") }
func (_ UnionStringLong) SetDouble(v float64) { panic("Unsupported operation") }
func (_ UnionStringLong) SetBytes(v []byte)   { panic("Unsupported operation") }
func (_ UnionStringLong) SetString(v string)  { panic("Unsupported operation") }

func (r *UnionStringLong) SetLong(v int64) {

	r.UnionType = (UnionStringLongTypeEnum)(v)
}

func (r *UnionStringLong) Get(i int) types.Field {

	switch i {
	case 0:
		return &types.String{Target: (&r.String)}
	case 1:
		return &types.Long{Target: (&r.Long)}
	}
	panic("Unknown field index")
}
func (_ UnionStringLong) NullField(i int)                  { panic("Unsupported operation") }
func (_ UnionStringLong) HintSize(i int)                   { panic("Unsupported operation") }
func (_ UnionStringLong) SetDefault(i int)                 { panic("Unsupported operation") }
func (_ UnionStringLong) AppendMap(key string) types.Field { panic("Unsupported operation") }
func (_ UnionStringLong) AppendArray() types.Field         { panic("Unsupported operation") }
func (_ UnionStringLong) Finalize()                        {}

func (r UnionStringLong) MarshalJSON() ([]byte, error) {

	switch r.UnionType {
	case UnionStringLongTypeEnumString:
		return json.Marshal(map[string]interface{}{"string": r.String})
	case UnionStringLongTypeEnumLong:
		return json.Marshal(map[string]interface{}{"long": r.Long})
	}
	return nil, fmt.Errorf("invalid value for UnionStringLong")
}

func (r *UnionStringLong) UnmarshalJSON(data []byte) error {

	var fields map[string]json.RawMessage
	if err := json.Unmarshal(data, &fields); err != nil {
		return err
	}
	if len(fields) > 1 {
		return fmt.Errorf("more than one type supplied for union")
	}
	if value, ok := fields["string"]; ok {
		r.UnionType = 0
		return json.Unmarshal([]byte(value), &r.String)
	}
	if value, ok := fields["long"]; ok {
		r.UnionType = 1
		return json.Unmarshal([]byte(value), &r.Long)
	}
	return fmt.Errorf("invalid value for UnionStringLong")
}

test_event.go

// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT.
/*
 * SOURCE:
 *     TestEvent.avsc
 */
package avro

import (
	"encoding/json"
	"fmt"
	"io"

	"github.com/actgardner/gogen-avro/v10/compiler"
	"github.com/actgardner/gogen-avro/v10/vm"
	"github.com/actgardner/gogen-avro/v10/vm/types"
)

var _ = fmt.Printf

// Test event
type TestEvent struct {
	Exampleunion UnionStringLong `json:"exampleunion"`
}

const TestEventAvroCRC64Fingerprint = "Ve\xa3m\xc8@\t\x9b"

func NewTestEvent() TestEvent {
	r := TestEvent{}
	r.Exampleunion = NewUnionStringLong()

	return r
}

func DeserializeTestEvent(r io.Reader) (TestEvent, error) {
	t := NewTestEvent()
	deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema()))
	if err != nil {
		return t, err
	}

	err = vm.Eval(r, deser, &t)
	return t, err
}

func DeserializeTestEventFromSchema(r io.Reader, schema string) (TestEvent, error) {
	t := NewTestEvent()

	deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema()))
	if err != nil {
		return t, err
	}

	err = vm.Eval(r, deser, &t)
	return t, err
}

func writeTestEvent(r TestEvent, w io.Writer) error {
	var err error
	err = writeUnionStringLong(r.Exampleunion, w)
	if err != nil {
		return err
	}
	return err
}

func (r TestEvent) Serialize(w io.Writer) error {
	return writeTestEvent(r, w)
}

func (r TestEvent) Schema() string {
	return "{\"doc\":\"Test event\",\"fields\":[{\"name\":\"exampleunion\",\"type\":[\"string\",\"long\"]}],\"name\":\"com.test.avro.TestEvent\",\"type\":\"record\"}"
}

func (r TestEvent) SchemaName() string {
	return "com.test.avro.TestEvent"
}

func (_ TestEvent) SetBoolean(v bool)    { panic("Unsupported operation") }
func (_ TestEvent) SetInt(v int32)       { panic("Unsupported operation") }
func (_ TestEvent) SetLong(v int64)      { panic("Unsupported operation") }
func (_ TestEvent) SetFloat(v float32)   { panic("Unsupported operation") }
func (_ TestEvent) SetDouble(v float64)  { panic("Unsupported operation") }
func (_ TestEvent) SetBytes(v []byte)    { panic("Unsupported operation") }
func (_ TestEvent) SetString(v string)   { panic("Unsupported operation") }
func (_ TestEvent) SetUnionElem(v int64) { panic("Unsupported operation") }

func (r *TestEvent) Get(i int) types.Field {
	switch i {
	case 0:
		r.Exampleunion = NewUnionStringLong()

		w := types.Record{Target: &r.Exampleunion}

		return w

	}
	panic("Unknown field index")
}

func (r *TestEvent) SetDefault(i int) {
	switch i {
	}
	panic("Unknown field index")
}

func (r *TestEvent) NullField(i int) {
	switch i {
	}
	panic("Not a nullable field index")
}

func (_ TestEvent) AppendMap(key string) types.Field { panic("Unsupported operation") }
func (_ TestEvent) AppendArray() types.Field         { panic("Unsupported operation") }
func (_ TestEvent) HintSize(int)                     { panic("Unsupported operation") }
func (_ TestEvent) Finalize()                        {}

func (_ TestEvent) AvroCRC64Fingerprint() []byte {
	return []byte(TestEventAvroCRC64Fingerprint)
}

func (r TestEvent) MarshalJSON() ([]byte, error) {
	var err error
	output := make(map[string]json.RawMessage)
	output["exampleunion"], err = json.Marshal(r.Exampleunion)
	if err != nil {
		return nil, err
	}
	return json.Marshal(output)
}

func (r *TestEvent) UnmarshalJSON(data []byte) error {
	var fields map[string]json.RawMessage
	if err := json.Unmarshal(data, &fields); err != nil {
		return err
	}

	var val json.RawMessage
	val = func() json.RawMessage {
		if v, ok := fields["exampleunion"]; ok {
			return v
		}
		return nil
	}()

	if val != nil {
		if err := json.Unmarshal([]byte(val), &r.Exampleunion); err != nil {
			return err
		}
	} else {
		return fmt.Errorf("no value specified for exampleunion")
	}
	return nil
}

[petedannemann]

For this library, I was envisioning the UnionConverter code being generated in a similar way to what gogen-avro does

[nrwiersma]

If you all want to put the time in to make this work, it would be considered.