diff --git a/README.md b/README.md index 80a84eb..8f4b62c 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ There are a couple of subtle ways you can configure the encoders. - `,raw`, which allows byteslice-like items (like `[]byte` and `string`) to be written to the buffer directly with no conversion, quoting or otherwise. `nil` or empty fields annotated as `raw` will output `null`. - `,encoder` which instead of the standard serialization method for a given type, nominates that its `.JSONEncode(*jingo.Buffer)` function or `EncodeJSON(io.Writer)` function are invoked instead. From there you can manually write to the buffer or writer for that particular field. There are a choice of 2 interfaces you need to comply with depending on your use case, either `jingo.JSONEncoder` (which introduces a dependency on `Buffer`), or `jingo.JSONMarshaler` which allows writing directly to an `io.Writer`. - `,escape`, which safely escapes `"`,`\`, line feed (`\n`), carriage return (`\r`) and tab (`\t`) characters to valid JSON whilst writing. To get the same functionality when using `SliceEncoder` on its own, use `jingo.EscapeString` to initialize the encoder - e.g `NewSliceEncoder([]jingo.EscapeString)` - instead of `string` directly. There is obviously a performance impact on the write speed using this option, the benchmarks show it takes twice the time of a standard string write, so whilst it is still faster than using the stdlib, to get the best performance it is recommended to only be used when needed and only then when the escaping work can't be done up-front. + - `,omitempty`, this will omit the field from being output excluding structs. This has a performance impact and should be used when required. This is still more performant than the stdlib. ## How does it work @@ -114,7 +115,6 @@ As part of the instruction set compilation it also generates static meta-data, i The package is designed to be performant and as such it is not 100% functionally compatible with stdlib. Specifically. -* 'Omit if empty' isn't supported, due to the nature of the instruction based approach we would be paying a performance price by including this - although it is not impossible with further effort. It isn't something that affects us as it can generally be worked around. * The `,string` tag option isn't supported, only strings are quoted by default - use `,stringer` instead to achieve the same results. This may be added in future releases. * Maps are currently not supported. Initial thoughts were given that this is a performance focused library it doesn't make much sense to iterate maps and would advise against doing so for performance sensitive applications - **however - maps are being added**! diff --git a/doc.go b/doc.go index 541e80f..f023d97 100644 --- a/doc.go +++ b/doc.go @@ -1,12 +1,14 @@ -// Package jingo provides the ability to encode go structs to a buffer as JSON. -// -// The main take-aways are -// It's very fast. -// Very low allocs, 0 in a lot of cases. -// Clear API - similar to the stdlib. It just uses struct tags. -// No other library dependencies. -// It doesn't require a build step, like `go generate`. -// -// You only need to create an instance of an encoder once per struct/slice type -// you wish to marshal. +/* +Package jingo provides the ability to encode go structs to a buffer as JSON. + +The main features are +It's very fast. +Very low allocs, 0 in a lot of cases. +Clear API - similar to the stdlib. It just uses struct tags. +No other library dependencies. +It doesn't require a build step, like `go generate`. + +You only need to create an instance of an encoder once per struct/slice type +you wish to marshal. +*/ package jingo diff --git a/jingo_test.go b/jingo_test.go index 17dc1a8..fb81dbc 100644 --- a/jingo_test.go +++ b/jingo_test.go @@ -12,31 +12,81 @@ import ( ) type all struct { - PropBool bool `json:"propBool"` - PropInt int `json:"propInt"` - PropInt8 int8 `json:"propInt8"` - PropInt16 int16 `json:"propInt16"` - PropInt32 int32 `json:"propInt32"` - PropInt64 int64 `json:"propInt64"` - PropUint uint `json:"propUint"` - PropUint8 uint8 `json:"propUint8"` - PropUint16 uint16 `json:"propUint16"` - PropUint32 uint32 `json:"propUint32"` - PropUint64 uint64 `json:"propUint64"` - PropFloat32 float32 `json:"propFloat32"` - PropFloat64 float64 `json:"propFloat64,stringer"` - PropString string `json:"propString"` - PropStruct struct { + ignoreMe1 string + PropBool bool `json:"propBool"` + PropInt int `json:"propInt"` + PropInt8 int8 `json:"propInt8"` + PropInt16 int16 `json:"propInt16"` + PropInt32 int32 `json:"propInt32"` + PropInt64 int64 `json:"propInt64"` + PropUint uint `json:"propUint"` + PropUint8 uint8 `json:"propUint8"` + PropUint16 uint16 `json:"propUint16"` + PropUint32 uint32 `json:"propUint32"` + PropUint64 uint64 `json:"propUint64"` + PropFloat32 float32 `json:"propFloat32"` + PropFloat64 float64 `json:"propFloat64,stringer"` + PropString string `json:"propString"` + ignoreStruct struct { + ignoreMeStruct1 string + ignoreMeStruct2 string + ignoreMeStruct3 string + } + PropStruct struct { PropNames []string `json:"propName"` PropPs []*string `json:"ps"` PropNamesEscaped []string `json:"propNameEscaped,escape"` } `json:"propStruct"` - PropEncode encode0 `json:"propEncode,encoder"` - PropEncodeP *encode0 `json:"propEncodeP,encoder"` - PropEncodenilP *encode0 `json:"propEncodenilP,encoder"` - PropEncodeS encode1 `json:"propEncodeS,encoder"` - PropJSONMarshaler jsonMarshaler `json:"propJSONMarshaler,encoder"` + PropEncode encode0 `json:"propEncode,encoder"` + PropEncodeP *encode0 `json:"propEncodeP,encoder"` + PropEncodenilP *encode0 `json:"propEncodenilP,encoder"` + PropEncodeS encode1 `json:"propEncodeS,encoder"` + PropJSONMarshaler jsonMarshaler `json:"propJSONMarshaler,encoder"` + ignoreMe2 string PropJSONMarshalerP *jsonMarshaler `json:"propJSONMarshalerP,encoder"` + ignoreMe3 string +} + +type allSubOmit struct { + PropBool bool `json:"propBool,omitempty"` + PropInt int `json:"propInt,omitempty"` + PropString string `json:"propString,omitempty"` +} + +type allOmit struct { + ignoreMe1 string + PropBool bool `json:"propBool,omitempty"` + PropInt int `json:"propInt,omitempty"` + PropInt8 int8 `json:"propInt8,omitempty"` + PropInt16 int16 `json:"propInt16,omitempty"` + PropInt32 int32 `json:"propInt32,omitempty"` + PropInt64 int64 `json:"propInt64,omitempty"` + PropUint uint `json:"propUint,omitempty"` + PropUint8 uint8 `json:"propUint8,omitempty"` + PropUint16 uint16 `json:"propUint16,omitempty"` + PropUint32 uint32 `json:"propUint32,omitempty"` + PropUint64 uint64 `json:"propUint64,omitempty"` + PropFloat32 float32 `json:"propFloat32,omitempty"` + PropFloat64 float64 `json:"propFloat64,stringer,omitempty"` + PropString string `json:"propString,omitempty"` + PropStruct allSubOmit `json:"propStruct,omitempty"` + ignoreStruct struct { + ignoreMeStruct1 string + ignoreMeStruct2 string + ignoreMeStruct3 string + } + PropPointerStruct *allSubOmit `json:"propPointerStruct,omitempty"` + PropSlice []string `json:"propName,omitempty"` + PropPointerSlice []*string `json:"ps,omitempty"` + PropSliceEscaped []string `json:"propNameEscaped,escape,omitempty"` + PropEncode encodeOmit0 `json:"propEncode,encoder,omitempty"` + PropEncodeP *encodeOmit0 `json:"propEncodeP,encoder,omitempty"` + PropEncodenilP *encodeOmit0 `json:"propEncodenilP,encoder,omitempty"` + PropEncodeS encodeOmit1 `json:"propEncodeS,encoder,omitempty"` + PropJSONMarshaler jsonMarshalerOmit `json:"propJSONMarshaler,encoder,omitempty"` + ignoreMe3 string + PropJSONMarshalerP *jsonMarshalerOmit `json:"propJSONMarshalerP,encoder,omitempty"` + ignoreMe2 string } type encode0 struct { @@ -58,6 +108,32 @@ func (e *encode1) JSONEncode(w *Buffer) { } } +type encodeOmit0 struct { + val byte +} + +func (e *encodeOmit0) JSONEncode(w *Buffer) { + if e.val == 0 { + w.Write([]byte{'{', '}'}) + return + } + + w.WriteByte(e.val) +} + +type encodeOmit1 []encodeOmit0 + +func (e *encodeOmit1) JSONEncode(w *Buffer) { + if len(*e) == 0 { + w.Write([]byte{'{', '}'}) + return + } + + for _, v := range *e { + w.WriteByte(v.val) + } +} + type jsonMarshaler struct { val []byte } @@ -66,6 +142,18 @@ func (j *jsonMarshaler) EncodeJSON(w io.Writer) { w.Write(j.val) } +type jsonMarshalerOmit struct { + val []byte +} + +func (j *jsonMarshalerOmit) EncodeJSON(w io.Writer) { + if len(j.val) == 0 { + w.Write([]byte{'{', '}'}) + return + } + w.Write(j.val) +} + func Example() { enc := NewStructEncoder(all{}) @@ -101,6 +189,16 @@ func Example() { PropEncodeS: encode1{encode0{'3'}, encode0{'4'}}, PropJSONMarshaler: jsonMarshaler{[]byte("1")}, PropJSONMarshalerP: &jsonMarshaler{[]byte("2")}, + + ignoreMe1: "1", + ignoreMe2: "2", + ignoreMe3: "3", + + ignoreStruct: struct { + ignoreMeStruct1 string + ignoreMeStruct2 string + ignoreMeStruct3 string + }{ignoreMeStruct1: "", ignoreMeStruct2: "", ignoreMeStruct3: ""}, }, b) fmt.Println(b.String()) @@ -258,7 +356,7 @@ func BenchmarkUnicode(b *testing.B) { var enc = NewStructEncoder(UnicodeObject{}) - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { buf := NewBufferFromPool() enc.Marshal(&ub, buf) @@ -273,7 +371,7 @@ func BenchmarkUnicodeStdLib(b *testing.B) { Russian: "ру́сский язы́к", } - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { json.Marshal(&ub) } @@ -331,7 +429,7 @@ func BenchmarkUnicodeLarge(b *testing.B) { var enc = NewStructEncoder(UnicodeObjectLarge{}) - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { buf := NewBufferFromPool() enc.Marshal(&ub, buf) @@ -350,7 +448,7 @@ func BenchmarkUnicodeLargeStdLib(b *testing.B) { Test3: "ascdjkl ascdhjklacdshlacdshjkl acdshjcdhjkl acdshjl kacdshjkl acdshjkacdshjklacdhjskl hjkl acdshjkl acdshjkl acdshjkl acdshjkl acdshjkl acdshjk lacdshjk acdshjkl acdshjkl hjkl acdshjkl acdshjkl acdshjkl cdshjkl acdshjkl acdshjkl acdshjkl acdshjkl acdshjkl acdshjkl acdshjkl ", } - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { json.Marshal(&ub) } @@ -440,6 +538,367 @@ func BenchmarkTimeStdLib(b *testing.B) { } } +func TestStructOmitempty(t *testing.T) { + + enc5 := NewStructEncoder(allOmit{}) + + type marshaler interface { + Marshal(s interface{}, w *Buffer) + } + + tests := []struct { + name string + enc marshaler + v interface{} + want []byte + json bool + }{ + { + "No Omit", + enc5, + &allOmit{ + PropBool: true, + PropInt: 1234567878910111212, + PropInt8: 123, + PropInt16: 12349, + PropInt32: 1234567891, + PropInt64: 1234567878910111213, + PropUint: 12345678789101112138, + PropUint8: 255, + PropUint16: 12345, + PropUint32: 1234567891, + PropUint64: 12345678789101112139, + PropFloat32: 21.232426, + PropFloat64: 2799999999888.28293031999999, + PropString: "thirty two thirty four", + PropStruct: allSubOmit{ + PropBool: true, + PropInt: 1234567878910111212, + PropString: "thirty two thirty four", + }, + PropPointerStruct: &allSubOmit{ + PropBool: true, + PropInt: 1234567878910111212, + PropString: "thirty two thirty four", + }, + PropSlice: []string{"a name", "another name", "another"}, + PropPointerSlice: []*string{&s, nil, &s}, + PropSliceEscaped: []string{"one\\two\\,three\"", "\"four\\five\\,six\""}, + PropEncode: encodeOmit0{'1'}, + PropEncodeP: &encodeOmit0{'2'}, + PropEncodeS: encodeOmit1{encodeOmit0{'3'}, encodeOmit0{'4'}}, + PropJSONMarshaler: jsonMarshalerOmit{[]byte("1")}, + PropJSONMarshalerP: &jsonMarshalerOmit{[]byte("2")}, + + ignoreMe1: "1", + ignoreMe2: "2", + ignoreMe3: "3", + + ignoreStruct: struct { + ignoreMeStruct1 string + ignoreMeStruct2 string + ignoreMeStruct3 string + }{ignoreMeStruct1: "", ignoreMeStruct2: "", ignoreMeStruct3: ""}, + }, + []byte(`{"propBool":true,"propInt":1234567878910111212,"propInt8":123,"propInt16":12349,"propInt32":1234567891,"propInt64":1234567878910111213,"propUint":12345678789101112138,"propUint8":255,"propUint16":12345,"propUint32":1234567891,"propUint64":12345678789101112139,"propFloat32":21.232426,"propFloat64":2799999999888.2827,"propString":"thirty two thirty four","propStruct":{"propBool":true,"propInt":1234567878910111212,"propString":"thirty two thirty four"},"propPointerStruct":{"propBool":true,"propInt":1234567878910111212,"propString":"thirty two thirty four"},"propName":["a name","another name","another"],"ps":["test pointer string b",null,"test pointer string b"],"propNameEscaped":["one\\two\\,three\"","\"four\\five\\,six\""],"propEncode":1,"propEncodeP":2,"propEncodeS":34,"propJSONMarshaler":1,"propJSONMarshalerP":2}`), + false, + }, + { + "Omit", + enc5, + &allOmit{}, + []byte(`{"propStruct":{},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Empty Omit", + enc5, + &allOmit{ + PropBool: false, + PropInt: 0, + PropInt8: 0, + PropInt16: 0, + PropInt32: 0, + PropInt64: 0, + PropUint: 0, + PropUint8: 0, + PropUint16: 0, + PropUint32: 0, + PropUint64: 0, + PropFloat32: 0, + PropFloat64: 0, + PropString: "", + PropStruct: allSubOmit{}, + PropPointerStruct: &allSubOmit{}, + PropSlice: []string{}, + PropPointerSlice: []*string{}, + PropSliceEscaped: []string{}, + PropEncode: encodeOmit0{}, + PropEncodeP: &encodeOmit0{}, + PropEncodeS: encodeOmit1{}, + PropJSONMarshaler: jsonMarshalerOmit{}, + PropJSONMarshalerP: &jsonMarshalerOmit{}, + }, + []byte(`{"propStruct":{},"propPointerStruct":{},"propEncode":{},"propEncodeP":{},"propJSONMarshaler":{},"propJSONMarshalerP":{}}`), + true, + }, + { + "Omit Except String", + enc5, + &allOmit{ + PropString: "noomit", + }, + []byte(`{"propString":"noomit","propStruct":{},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Int", + enc5, + &allOmit{ + PropInt: 365, + }, + []byte(`{"propInt":365,"propStruct":{},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Struct", + enc5, + &allOmit{ + PropStruct: allSubOmit{ + PropBool: true, + PropInt: 1234567878910111212, + PropString: "thirty two thirty four", + }, + }, + []byte(`{"propStruct":{"propBool":true,"propInt":1234567878910111212,"propString":"thirty two thirty four"},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Struct With Bool", + enc5, + &allOmit{ + PropStruct: allSubOmit{ + PropBool: true, + }, + }, + []byte(`{"propStruct":{"propBool":true},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Struct With String", + enc5, + &allOmit{ + PropStruct: allSubOmit{ + PropString: "thirty two thirty four", + }, + }, + []byte(`{"propStruct":{"propString":"thirty two thirty four"},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Struct With Int", + enc5, + &allOmit{ + PropStruct: allSubOmit{ + PropInt: 1234567878910111212, + }, + }, + []byte(`{"propStruct":{"propInt":1234567878910111212},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Slice", + enc5, + &allOmit{ + PropSlice: []string{"a name", "another name", "another"}, + }, + []byte(`{"propStruct":{},"propName":["a name","another name","another"],"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Pointer Slice", + enc5, + allOmit{ + PropPointerSlice: []*string{&s, nil, &s}, + }, + []byte(`{"propStruct":{},"ps":["test pointer string b",null,"test pointer string b"],"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Pointer Struct", + enc5, + &allOmit{ + PropPointerStruct: &allSubOmit{ + PropBool: true, + PropInt: 1234567878910111212, + PropString: "thirty two thirty four", + }, + }, + []byte(`{"propStruct":{},"propPointerStruct":{"propBool":true,"propInt":1234567878910111212,"propString":"thirty two thirty four"},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Pointer Struct With String", + enc5, + &allOmit{ + PropPointerStruct: &allSubOmit{ + PropString: "thirty two thirty four", + }, + }, + []byte(`{"propStruct":{},"propPointerStruct":{"propString":"thirty two thirty four"},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + { + "Omit Except Pointer Struct With Int", + enc5, + &allOmit{ + PropPointerStruct: &allSubOmit{ + PropInt: 1234567878910111212, + }, + }, + []byte(`{"propStruct":{},"propPointerStruct":{"propInt":1234567878910111212},"propEncode":{},"propJSONMarshaler":{}}`), + true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + buf := NewBufferFromPool() + defer buf.ReturnToPool() + + tt.enc.Marshal(tt.v, buf) + + j, _ := json.Marshal(tt.v) + + if !bytes.Equal(tt.want, buf.Bytes) || (tt.json && !bytes.Equal(j, buf.Bytes)) { + t.Errorf("\nwant:\n%s\ngot:\n%s\njson:\n%s\n", tt.want, buf.Bytes, j) + } + }) + } +} + +type omitBenchmark struct { + PropBool bool `json:"propBool,omitempty"` + PropInt int `json:"propInt,omitempty"` + PropInt8 int8 `json:"propInt8,omitempty"` + PropInt16 int16 `json:"propInt16,omitempty"` + PropInt32 int32 `json:"propInt32,omitempty"` + PropInt64 int64 `json:"propInt64,omitempty"` + PropUint uint `json:"propUint,omitempty"` + PropUint8 uint8 `json:"propUint8,omitempty"` + PropUint16 uint16 `json:"propUint16,omitempty"` + PropUint32 uint32 `json:"propUint32,omitempty"` + PropUint64 uint64 `json:"propUint64,omitempty"` + PropFloat32 float32 `json:"propFloat32,omitempty"` + PropFloat64 float64 `json:"propFloat64,stringer,omitempty"` + PropString string `json:"propString,omitempty"` + PropPointerStruct *allSubOmit `json:"propPointerStruct,omitempty"` + PropSlice []string `json:"propName,omitempty"` + PropPointerSlice []*string `json:"ps,omitempty"` + PropSliceEscaped []string `json:"propNameEscaped,escape,omitempty"` + PropEncodeP *encode0 `json:"propEncodeP,encoder,omitempty"` + PropEncodenilP *encode0 `json:"propEncodenilP,encoder,omitempty"` + PropJSONMarshalerP *jsonMarshaler `json:"propJSONMarshalerP,encoder,omitempty"` +} + +var omitBench = omitBenchmark{} + +func BenchmarkOmitEmpty(b *testing.B) { + var enc = NewStructEncoder(omitBenchmark{}) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf := NewBufferFromPool() + enc.Marshal(&omitBench, buf) + buf.ReturnToPool() + } +} + +func BenchmarkOmitEmptyStdLib(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + json.Marshal(&omitBench) + } +} + +var omitBenchWithData = omitBenchmark{ + PropBool: true, + PropInt: 1234567878910111212, + PropInt8: 123, + PropInt16: 12349, + PropInt32: 1234567891, + PropInt64: 1234567878910111213, + PropUint: 12345678789101112138, + PropUint8: 255, + PropUint16: 12345, + PropUint32: 1234567891, + PropUint64: 12345678789101112139, + PropFloat32: 21.232426, + PropFloat64: 2799999999888.28293031999999, + PropString: "thirty two thirty four", + PropPointerStruct: &allSubOmit{ + PropBool: true, + PropInt: 1234567878910111212, + PropString: "thirty two thirty four", + }, + PropSlice: []string{"a name", "another name", "another"}, + PropPointerSlice: []*string{&s, nil, &s}, + PropSliceEscaped: []string{"one\\two\\,three\"", "\"four\\five\\,six\""}, + PropEncodeP: &encode0{'2'}, + PropJSONMarshalerP: &jsonMarshaler{[]byte("2")}, +} + +func BenchmarkOmitEmptyNonEmpty(b *testing.B) { + var enc = NewStructEncoder(omitBenchmark{}) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf := NewBufferFromPool() + enc.Marshal(&omitBenchWithData, buf) + buf.ReturnToPool() + } +} + +func BenchmarkOmitEmptyNonEmptyStdLib(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + json.Marshal(&omitBenchWithData) + } +} + +type OmitSmallPayload struct { + St int `json:"st,omitempty"` + Sid int `json:"sid,omitempty"` + Tt string `json:"tt,omitempty"` + Gr int `json:"gr,omitempty"` + UUID string `json:"uuid,omitempty"` + IP string `json:"ip,omitempty"` + Ua string `json:"ua,omitempty"` + Tz int `json:"tz,omitempty"` + V int `json:"v,omitempty"` +} + +var omitSmallBench = OmitSmallPayload{} + +func BenchmarkOmitEmptySmall(b *testing.B) { + var enc = NewStructEncoder(OmitSmallPayload{}) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf := NewBufferFromPool() + enc.Marshal(&omitSmallBench, buf) + buf.ReturnToPool() + } +} + +func BenchmarkOmitEmptySmallStdLib(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + json.Marshal(&omitSmallBench) + } +} + func TestSliceEncoder(t *testing.T) { enc := NewSliceEncoder([]string{}) @@ -502,7 +961,7 @@ func BenchmarkSlice(b *testing.B) { var enc = NewSliceEncoder([]string{}) - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { buf := NewBufferFromPool() enc.Marshal(&ss, buf) @@ -522,7 +981,7 @@ func BenchmarkSliceEscape(b *testing.B) { var enc = NewSliceEncoder([]EscapeString{}) - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { buf := NewBufferFromPool() enc.Marshal(&ss, buf) @@ -539,7 +998,7 @@ func BenchmarkSliceStdLib(b *testing.B) { "last one, promise", } - b.StartTimer() + b.ResetTimer() for i := 0; i < b.N; i++ { json.Marshal(&ss) } diff --git a/ptrconvert.go b/ptrconvert.go index a08f8de..b8cf2f8 100755 --- a/ptrconvert.go +++ b/ptrconvert.go @@ -88,6 +88,7 @@ func ptrFloat64ToBuf(v unsafe.Pointer, b *Buffer) { b.Bytes = strconv.AppendFloat(b.Bytes, *(*float64)(v), 'f', -1, 64) } +//go:nocheckptr func ptrStringToBuf(v unsafe.Pointer, b *Buffer) { b.WriteString(*(*string)(v)) } @@ -138,3 +139,24 @@ func ptrEscapeStringToBuf(v unsafe.Pointer, w *Buffer) { w.WriteString(bs[pos:]) } } + +var getIsZeroFunc = map[reflect.Kind]func(unsafe.Pointer) bool{ + reflect.Ptr: func(v unsafe.Pointer) bool { return *(*unsafe.Pointer)(v) == nil }, + reflect.Slice: func(v unsafe.Pointer) bool { return (*(*sliceHeader)(v)).Len == 0 }, + reflect.Bool: func(v unsafe.Pointer) bool { return !(*(*bool)(v)) }, + reflect.String: func(v unsafe.Pointer) bool { return *(*string)(v) == "" }, + reflect.Int: func(v unsafe.Pointer) bool { return *(*int)(v) == 0 }, + reflect.Int8: func(v unsafe.Pointer) bool { return *(*int8)(v) == 0 }, + reflect.Int16: func(v unsafe.Pointer) bool { return *(*int16)(v) == 0 }, + reflect.Int32: func(v unsafe.Pointer) bool { return *(*int32)(v) == 0 }, + reflect.Int64: func(v unsafe.Pointer) bool { return *(*int64)(v) == 0 }, + reflect.Uint: func(v unsafe.Pointer) bool { return *(*uint)(v) == 0 }, + reflect.Uint8: func(v unsafe.Pointer) bool { return *(*uint8)(v) == 0 }, + reflect.Uint16: func(v unsafe.Pointer) bool { return *(*uint16)(v) == 0 }, + reflect.Uint32: func(v unsafe.Pointer) bool { return *(*uint32)(v) == 0 }, + reflect.Uint64: func(v unsafe.Pointer) bool { return *(*uint64)(v) == 0 }, + reflect.Float32: func(v unsafe.Pointer) bool { return *(*float32)(v) == 0 }, + reflect.Float64: func(v unsafe.Pointer) bool { return *(*float64)(v) == 0 }, + reflect.Complex64: func(v unsafe.Pointer) bool { return *(*complex64)(v) == 0 }, + reflect.Complex128: func(v unsafe.Pointer) bool { return *(*complex128)(v) == complex(0, 0) }, +} diff --git a/structencoder.go b/structencoder.go index df18d77..ca1c276 100755 --- a/structencoder.go +++ b/structencoder.go @@ -20,10 +20,12 @@ import ( // static, leapFun/offset, fun are mutually exclusive. we've used a concrete type for speed. type instruction struct { static []byte // provides a fast path for writing static chunks without needing an instruction function + sub []instruction // sub instructions for omitempty kind int // used to switch special paths in Marshal, like string fast path offset uintptr // used in conjunction with leapFun leapFun func(unsafe.Pointer, *Buffer) // provides a fast path for simple write & avoids wrapping function to capture offset fun func(unsafe.Pointer, *Buffer) // full instruction function for when the approaches above fail + isZero func(unsafe.Pointer) bool // isZero tests if pointer is zero } const ( @@ -31,6 +33,7 @@ const ( kindStringField kindStatic kindInt + KindOmit ) // iface describes the memory footprint of interface{} @@ -47,6 +50,7 @@ type StructEncoder struct { i int // iter cb Buffer // side buffer for static data cpos int // side buffer position + o bool // field is omitempty } // Marshal executes the instructions for a given type and writes the resulting @@ -56,7 +60,6 @@ func (e *StructEncoder) Marshal(s interface{}, w *Buffer) { p := (*(*iface)(unsafe.Pointer(&s))).Data for i := 0; i < len(e.instructions); i++ { - if e.instructions[i].kind == kindStatic { // static data fast path w.Write(e.instructions[i].static) continue @@ -68,6 +71,36 @@ func (e *StructEncoder) Marshal(s interface{}, w *Buffer) { continue } else if e.instructions[i].leapFun != nil { // simple 'conv' function fast path e.instructions[i].leapFun(unsafe.Pointer(uintptr(p)+e.instructions[i].offset), w) + continue + } else if e.instructions[i].kind == KindOmit { + // this is managed by omitFlunk + w.Write(e.instructions[i].static) + if e.instructions[i].isZero(unsafe.Pointer(uintptr(p) + e.instructions[i].offset)) { + // this is managed by setupLastField + if e.instructions[i].fun != nil { + e.instructions[i].fun(nil, w) + } + continue + } + + for si := 0; si < len(e.instructions[i].sub); si++ { + if e.instructions[i].sub[si].kind == kindStatic { // static data fast path + w.Write(e.instructions[i].sub[si].static) + continue + } else if e.instructions[i].sub[si].kind == kindStringField { // string fields fast path, allows inlining of whole write + ptrStringToBuf(unsafe.Pointer(uintptr(p)+e.instructions[i].sub[si].offset), w) + continue + } else if e.instructions[i].sub[si].kind == kindInt { // int fields fast path, allows inlining of whole write + ptrIntToBuf(unsafe.Pointer(uintptr(p)+e.instructions[i].sub[si].offset), w) + continue + } else if e.instructions[i].sub[si].leapFun != nil { // simple 'conv' function fast path + e.instructions[i].sub[si].leapFun(unsafe.Pointer(uintptr(p)+e.instructions[i].sub[si].offset), w) + continue + } + + e.instructions[i].sub[si].fun(p, w) // all other instruction types + } + continue } @@ -83,7 +116,6 @@ func NewStructEncoder(t interface{}) *StructEncoder { e.chunk("{") - emit := 0 // track number of fields we emit // pass over each field in the struct to build up our instruction set for each for e.i = 0; e.i < tt.NumField(); e.i++ { e.f = tt.Field(e.i) @@ -92,12 +124,13 @@ func NewStructEncoder(t interface{}) *StructEncoder { if tag == "" { continue } - emit++ - // write the key - if emit > 1 { - e.chunk(",") + e.o = false + e.o = opts.Contains("omitempty") && e.f.Type.Kind() != reflect.Struct + if e.o { + e.omitFlunk() } + e.chunk(`"` + tag + `":`) switch { @@ -147,16 +180,71 @@ func NewStructEncoder(t interface{}) *StructEncoder { // create an instruction which reads from a standard field e.valueInst(e.f.Type.Kind(), e.val) } + + e.chunk(",") + + // if omitempty we need to flunk so just this field is omitted + if e.o { + e.flunk() + } } + e.setupLastField() // for comma handling e.chunk("}") + e.flunk() return e } -func (e *StructEncoder) appendInstructionFun(fun func(unsafe.Pointer, *Buffer)) { - e.instructions = append(e.instructions, instruction{fun: fun}) +// setupLastField manually removes the comma from the last field. +// +// If it is an omit instruction we know to remove the last byte always. +// We then add a function to the general fun field (that never gets used) to reset the comma if this field is omitted +// +// If it's not omitempty we check if the last item is a comma as it could be a '{' and remove the comma. +func (e *StructEncoder) setupLastField() { + if e.o { + instr := &e.instructions[len(e.instructions)-1] + subInstr := &instr.sub[len(instr.sub)-1] + subInstr.static = subInstr.static[:len(subInstr.static)-1] + + instr.fun = func(_ unsafe.Pointer, w *Buffer) { + if w.Bytes[len(w.Bytes)-1] == ',' { // remove any superfluous "," in the event the last field was omitted + w.Bytes = w.Bytes[:len(w.Bytes)-1] + } + } + e.o = false + + } else if e.cb.Bytes[len(e.cb.Bytes)-1] == ',' { + // it doesn't matter if this makes the slice empty as this is before the flunk + e.cb.Bytes = e.cb.Bytes[:len(e.cb.Bytes)-1] + } +} + +// appendInstruction is used to add an instruction to the decoder, we use this to ensure that omitEmpty is being set regardless of what instruction type is being passed through. +// If it is not an omit empty, it just appends it into the instructions list +// +// This also manages the omitempty functions on the parent instruction that allows us to perform isZero checks. +func (e *StructEncoder) appendInstruction(instr instruction) { + // if this is an omit field we append to the current instruction instead of appending a new one for the isZero check + if e.o { + omitInstr := &e.instructions[len(e.instructions)-1] + // if it is not a static kind, we know to also append the isZero check and offset + if instr.kind != kindStatic { + omitInstr.isZero = getIsZeroFunc[e.f.Type.Kind()] + omitInstr.offset = instr.offset + } + + omitInstr.sub = append(omitInstr.sub, instr) + return + } + + e.instructions = append(e.instructions, instr) +} + +func (e *StructEncoder) appendInstructionFun(fun func(unsafe.Pointer, *Buffer), offset uintptr) { + e.appendInstruction(instruction{fun: fun, offset: offset}) } func (e *StructEncoder) optInstrStringer() { @@ -218,6 +306,7 @@ func (e *StructEncoder) optInstrEncoderWriter() { w.Write(null) return } + e.EncodeJSON(w) } @@ -252,10 +341,11 @@ func (e *StructEncoder) optInstrEscape() { /// create an escape string encoder internally instead of mirroring the struct, so people only need to pass the ,escape opt instead enc := NewSliceEncoder([]EscapeString{}) f := e.f + e.appendInstructionFun(func(v unsafe.Pointer, w *Buffer) { var em interface{} = unsafe.Pointer(uintptr(v) + f.Offset) enc.Marshal(em, w) - }) + }, f.Offset) return } @@ -286,7 +376,22 @@ func (e *StructEncoder) flunk() { return } - e.instructions = append(e.instructions, instruction{static: bs, kind: kindStatic}) + e.appendInstruction(instruction{static: bs, kind: kindStatic}) +} + +// omitFlunk flushes whatever chunk data we've got buffered into a single instruction and exits/enters into omitMarshal +// this is to ensure that everything that needs to be written out is already complete before the isZero check +func (e *StructEncoder) omitFlunk() { + b := e.cb.Bytes + bs := b[e.cpos:] + e.cpos = len(b) + + if len(bs) == 0 { + e.instructions = append(e.instructions, instruction{kind: KindOmit}) + return + } + + e.instructions = append(e.instructions, instruction{static: bs, kind: KindOmit}) } // valueInst works out the conversion function we need for `k` and creates an instruction to write it to the buffer @@ -302,7 +407,7 @@ func (e *StructEncoder) valueInst(k reflect.Kind, instr func(func(unsafe.Pointer return } e.flunk() - e.instructions = append(e.instructions, instruction{offset: e.f.Offset, kind: kindInt}) + e.appendInstruction(instruction{offset: e.f.Offset, kind: kindInt}) case reflect.Bool, reflect.Int8, @@ -343,7 +448,7 @@ func (e *StructEncoder) valueInst(k reflect.Kind, instr func(func(unsafe.Pointer i := i e.appendInstructionFun(func(v unsafe.Pointer, w *Buffer) { conv(unsafe.Pointer(uintptr(v)+f.Offset+(uintptr(i)*offset)), w) - }) + }, f.Offset) } e.chunk("]") @@ -355,9 +460,8 @@ func (e *StructEncoder) valueInst(k reflect.Kind, instr func(func(unsafe.Pointer enc := NewSliceEncoder(reflect.ValueOf(e.t).Field(e.i).Interface()) f := e.f e.appendInstructionFun(func(v unsafe.Pointer, w *Buffer) { - var em interface{} = unsafe.Pointer(uintptr(v) + f.Offset) - enc.Marshal(em, w) - }) + enc.Marshal(unsafe.Pointer(uintptr(v)+f.Offset), w) + }, f.Offset) case reflect.String: @@ -372,7 +476,7 @@ func (e *StructEncoder) valueInst(k reflect.Kind, instr func(func(unsafe.Pointer /// fast path for strings e.flunk() // flush any chunk data we've buffered - e.instructions = append(e.instructions, instruction{offset: e.f.Offset, kind: kindStringField}) + e.appendInstruction(instruction{offset: e.f.Offset, kind: kindStringField}) e.chunk(`"`) case reflect.Struct: @@ -400,8 +504,9 @@ func (e *StructEncoder) valueInst(k reflect.Kind, instr func(func(unsafe.Pointer w.Write(null) return } + enc.Marshal(em, w) - }) + }, f.Offset) return } @@ -410,9 +515,8 @@ func (e *StructEncoder) valueInst(k reflect.Kind, instr func(func(unsafe.Pointer // now create another instruction which calls marshal on the struct, passing our writer f := e.f e.appendInstructionFun(func(v unsafe.Pointer, w *Buffer) { - var em interface{} = unsafe.Pointer(uintptr(v) + f.Offset) - enc.Marshal(em, w) - }) + enc.Marshal(unsafe.Pointer(uintptr(v)+f.Offset), w) + }, f.Offset) return case reflect.Invalid, @@ -433,7 +537,7 @@ func (e *StructEncoder) valueInst(k reflect.Kind, instr func(func(unsafe.Pointer func (e *StructEncoder) val(conv func(unsafe.Pointer, *Buffer)) { e.flunk() // flush any chunk data we've buffered - e.instructions = append(e.instructions, instruction{leapFun: conv, offset: e.f.Offset}) + e.appendInstruction(instruction{leapFun: conv, offset: e.f.Offset}) } // ptrval creates an instruction to read from a pointer field we're marshaling @@ -446,14 +550,13 @@ func (e *StructEncoder) ptrval(conv func(unsafe.Pointer, *Buffer)) { f := e.f e.appendInstructionFun(func(v unsafe.Pointer, w *Buffer) { - - p := unsafe.Pointer(*(*unsafe.Pointer)(unsafe.Pointer(uintptr(v) + f.Offset))) + p := *(*unsafe.Pointer)(unsafe.Pointer(uintptr(v) + f.Offset)) if p == unsafe.Pointer(nil) { w.Write(null) return } conv(p, w) - }) + }, f.Offset) } // ptrstringval is essentially the same as ptrval but quotes strings if not nil @@ -465,8 +568,7 @@ func (e *StructEncoder) ptrstringval(conv func(unsafe.Pointer, *Buffer)) { f := e.f e.appendInstructionFun(func(v unsafe.Pointer, w *Buffer) { - - p := unsafe.Pointer(*(*unsafe.Pointer)(unsafe.Pointer(uintptr(v) + f.Offset))) + p := *(*unsafe.Pointer)(unsafe.Pointer(uintptr(v) + f.Offset)) if p == unsafe.Pointer(nil) { w.Write(null) return @@ -476,7 +578,7 @@ func (e *StructEncoder) ptrstringval(conv func(unsafe.Pointer, *Buffer)) { w.WriteByte('"') conv(p, w) w.WriteByte('"') - }) + }, f.Offset) } // JSONEncoder works with the `.encoder` option. Fields can implement this to encode their own JSON string straight