From 659461c9efa5d3497b8261d1371c98cf2350e8a9 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 00:22:08 +0800 Subject: [PATCH 01/10] reset decoder when decoding is done to enable reuse of decoder --- .../decoder/decoder_bench_large_test.go | 6 +-- .../decoder/decoder_bench_medium_test.go | 6 +-- .../decoder/decoder_bench_small_test.go | 6 +-- decode.go | 43 +++++++++++++++---- decode_array.go | 2 + decode_bool.go | 6 ++- decode_interface.go | 3 ++ decode_number_float.go | 12 +++++- decode_number_int.go | 30 ++++++++++--- decode_number_uint.go | 24 +++++++++-- decode_object.go | 3 ++ decode_object_test.go | 2 +- decode_pool.go | 14 +++--- decode_sqlnull.go | 24 +++++++++-- decode_string.go | 6 ++- decode_time.go | 6 ++- decode_unsafe.go | 28 +++++++++--- 17 files changed, 173 insertions(+), 48 deletions(-) diff --git a/benchmarks/decoder/decoder_bench_large_test.go b/benchmarks/decoder/decoder_bench_large_test.go index 9902f8a..77dcaa9 100644 --- a/benchmarks/decoder/decoder_bench_large_test.go +++ b/benchmarks/decoder/decoder_bench_large_test.go @@ -10,7 +10,7 @@ import ( "github.com/mailru/easyjson" ) -func BenchmarkJsonParserDecodeObjLarge(b *testing.B) { +func BenchmarkJSONParserDecodeObjLarge(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { jsonparser.ArrayEach(benchmarks.LargeFixture, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { @@ -26,7 +26,7 @@ func BenchmarkJsonParserDecodeObjLarge(b *testing.B) { } } -func BenchmarkJsonIterDecodeObjLarge(b *testing.B) { +func BenchmarkJSONIterDecodeObjLarge(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { result := benchmarks.LargePayload{} @@ -34,7 +34,7 @@ func BenchmarkJsonIterDecodeObjLarge(b *testing.B) { } } -func BenchmarkEasyJsonDecodeObjLarge(b *testing.B) { +func BenchmarkEasyJSONDecodeObjLarge(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { result := benchmarks.LargePayload{} diff --git a/benchmarks/decoder/decoder_bench_medium_test.go b/benchmarks/decoder/decoder_bench_medium_test.go index d3f3622..51ef7a9 100644 --- a/benchmarks/decoder/decoder_bench_medium_test.go +++ b/benchmarks/decoder/decoder_bench_medium_test.go @@ -11,7 +11,7 @@ import ( "github.com/mailru/easyjson" ) -func BenchmarkJsonIterDecodeObjMedium(b *testing.B) { +func BenchmarkJSONIterDecodeObjMedium(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { result := benchmarks.MediumPayload{} @@ -36,14 +36,14 @@ func BenchmarkJSONParserDecodeObjMedium(b *testing.B) { } } -func BenchmarkEncodingJsonStructMedium(b *testing.B) { +func BenchmarkEncodingJSONStructMedium(b *testing.B) { for i := 0; i < b.N; i++ { var data = benchmarks.MediumPayload{} json.Unmarshal(benchmarks.MediumFixture, &data) } } -func BenchmarkEasyJsonDecodeObjMedium(b *testing.B) { +func BenchmarkEasyJSONDecodeObjMedium(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { result := benchmarks.MediumPayload{} diff --git a/benchmarks/decoder/decoder_bench_small_test.go b/benchmarks/decoder/decoder_bench_small_test.go index 7fe4bc2..70a5289 100644 --- a/benchmarks/decoder/decoder_bench_small_test.go +++ b/benchmarks/decoder/decoder_bench_small_test.go @@ -12,7 +12,7 @@ import ( "github.com/mailru/easyjson" ) -func BenchmarkJSONDecodeObjSmall(b *testing.B) { +func BenchmarkEncodingJSONObjSmall(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { result := benchmarks.SmallPayload{} @@ -36,7 +36,7 @@ func BenchmarkJSONParserSmall(b *testing.B) { } } -func BenchmarkJsonIterDecodeObjSmall(b *testing.B) { +func BenchmarkJSONIterDecodeObjSmall(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { result := benchmarks.SmallPayload{} @@ -44,7 +44,7 @@ func BenchmarkJsonIterDecodeObjSmall(b *testing.B) { } } -func BenchmarkEasyJsonDecodeObjSmall(b *testing.B) { +func BenchmarkEasyJSONDecodeObjSmall(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { result := benchmarks.SmallPayload{} diff --git a/decode.go b/decode.go index fbd07f7..2cd5130 100644 --- a/decode.go +++ b/decode.go @@ -13,17 +13,25 @@ import ( // overflows the target type, UnmarshalJSONArray skips that field and completes the unmarshaling as best it can. func UnmarshalJSONArray(data []byte, v UnmarshalerJSONArray) error { dec := borrowDecoder(nil, 0) - defer dec.Release() + + defer func() { + dec.reset() + dec.Release() + }() + dec.data = make([]byte, len(data)) copy(dec.data, data) dec.length = len(data) + _, err := dec.decodeArray(v) if err != nil { return err } + if dec.err != nil { return dec.err } + return nil } @@ -35,14 +43,21 @@ func UnmarshalJSONArray(data []byte, v UnmarshalerJSONArray) error { // overflows the target type, UnmarshalJSONObject skips that field and completes the unmarshaling as best it can. func UnmarshalJSONObject(data []byte, v UnmarshalerJSONObject) error { dec := borrowDecoder(nil, 0) - defer dec.Release() + + defer func() { + dec.reset() + dec.Release() + }() + dec.data = make([]byte, len(data)) copy(dec.data, data) dec.length = len(data) + _, err := dec.decodeObject(v) if err != nil { return err } + if dec.err != nil { return dec.err } @@ -215,11 +230,15 @@ func Unmarshal(data []byte, v interface{}) error { default: return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, vt)) } - defer dec.Release() - if err != nil { - return err + + if err == nil { + err = dec.err } - return dec.err + + dec.reset() + dec.Release() + + return err } // UnmarshalerJSONObject is the interface to implement to decode a JSON Object. @@ -257,7 +276,9 @@ func (dec *Decoder) Decode(v interface{}) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } + var err error + switch vt := v.(type) { case *string: err = dec.decodeString(vt) @@ -322,10 +343,14 @@ func (dec *Decoder) Decode(v interface{}) error { default: return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, vt)) } - if err != nil { - return err + + if err == nil { + err = dec.err } - return dec.err + + dec.reset() + + return err } // Non exported diff --git a/decode_array.go b/decode_array.go index 297f2ee..380ca2c 100644 --- a/decode_array.go +++ b/decode_array.go @@ -13,6 +13,8 @@ func (dec *Decoder) DecodeArray(v UnmarshalerJSONArray) error { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } _, err := dec.decodeArray(v) + dec.reset() + return err } func (dec *Decoder) decodeArray(arr UnmarshalerJSONArray) (int, error) { diff --git a/decode_bool.go b/decode_bool.go index 1dc304b..8328cc6 100644 --- a/decode_bool.go +++ b/decode_bool.go @@ -8,7 +8,11 @@ func (dec *Decoder) DecodeBool(v *bool) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeBool(v) + + err := dec.decodeBool(v) + dec.reset() + + return err } func (dec *Decoder) decodeBool(v *bool) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { diff --git a/decode_interface.go b/decode_interface.go index 015790d..19aa3ba 100644 --- a/decode_interface.go +++ b/decode_interface.go @@ -11,7 +11,10 @@ func (dec *Decoder) DecodeInterface(i *interface{}) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } + err := dec.decodeInterface(i) + dec.reset() + return err } diff --git a/decode_number_float.go b/decode_number_float.go index 0ac1a82..f7e2c50 100644 --- a/decode_number_float.go +++ b/decode_number_float.go @@ -7,7 +7,11 @@ func (dec *Decoder) DecodeFloat64(v *float64) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeFloat64(v) + + err := dec.decodeFloat64(v) + dec.reset() + + return err } func (dec *Decoder) decodeFloat64(v *float64) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { @@ -215,7 +219,11 @@ func (dec *Decoder) DecodeFloat32(v *float32) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeFloat32(v) + + err := dec.decodeFloat32(v) + dec.reset() + + return err } func (dec *Decoder) decodeFloat32(v *float32) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { diff --git a/decode_number_int.go b/decode_number_int.go index 9501904..c8b3de9 100644 --- a/decode_number_int.go +++ b/decode_number_int.go @@ -12,7 +12,11 @@ func (dec *Decoder) DecodeInt(v *int) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeInt(v) + + err := dec.decodeInt(v) + dec.reset() + + return err } func (dec *Decoder) decodeInt(v *int) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { @@ -119,7 +123,11 @@ func (dec *Decoder) DecodeInt16(v *int16) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeInt16(v) + + err := dec.decodeInt16(v) + dec.reset() + + return err } func (dec *Decoder) decodeInt16(v *int16) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { @@ -351,7 +359,11 @@ func (dec *Decoder) DecodeInt8(v *int8) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeInt8(v) + + err := dec.decodeInt8(v) + dec.reset() + + return err } func (dec *Decoder) decodeInt8(v *int8) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { @@ -582,7 +594,11 @@ func (dec *Decoder) DecodeInt32(v *int32) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeInt32(v) + + err := dec.decodeInt32(v) + dec.reset() + + return err } func (dec *Decoder) decodeInt32(v *int32) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { @@ -812,7 +828,11 @@ func (dec *Decoder) DecodeInt64(v *int64) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeInt64(v) + + err := dec.decodeInt64(v) + dec.reset() + + return err } func (dec *Decoder) decodeInt64(v *int64) error { diff --git a/decode_number_uint.go b/decode_number_uint.go index b57ef7a..5fa83ad 100644 --- a/decode_number_uint.go +++ b/decode_number_uint.go @@ -11,7 +11,11 @@ func (dec *Decoder) DecodeUint8(v *uint8) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeUint8(v) + + err := dec.decodeUint8(v) + dec.reset() + + return err } func (dec *Decoder) decodeUint8(v *uint8) error { @@ -123,7 +127,11 @@ func (dec *Decoder) DecodeUint16(v *uint16) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeUint16(v) + + err := dec.decodeUint16(v) + dec.reset() + + return err } func (dec *Decoder) decodeUint16(v *uint16) error { @@ -235,7 +243,11 @@ func (dec *Decoder) DecodeUint32(v *uint32) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeUint32(v) + + err := dec.decodeUint32(v) + dec.reset() + + return err } func (dec *Decoder) decodeUint32(v *uint32) error { @@ -347,7 +359,11 @@ func (dec *Decoder) DecodeUint64(v *uint64) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeUint64(v) + + err := dec.decodeUint64(v) + dec.reset() + + return err } func (dec *Decoder) decodeUint64(v *uint64) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { diff --git a/decode_object.go b/decode_object.go index 0fec9d2..84e6892 100644 --- a/decode_object.go +++ b/decode_object.go @@ -14,7 +14,10 @@ func (dec *Decoder) DecodeObject(j UnmarshalerJSONObject) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } + _, err := dec.decodeObject(j) + dec.reset() + return err } func (dec *Decoder) decodeObject(j UnmarshalerJSONObject) (int, error) { diff --git a/decode_object_test.go b/decode_object_test.go index 94f665c..fe0d741 100644 --- a/decode_object_test.go +++ b/decode_object_test.go @@ -1561,7 +1561,7 @@ func TestDecodeObjectPartial(t *testing.T) { "testSkipNumber": 123.23 }`) dec.length = len(dec.data) - err := dec.DecodeObject(&result) + _, err := dec.decodeObject(&result) assert.Nil(t, err, "err should be nil") assert.NotEqual(t, len(dec.data), dec.cursor) } diff --git a/decode_pool.go b/decode_pool.go index 68c5713..0d2f878 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -42,12 +42,7 @@ func BorrowDecoder(r io.Reader) *Decoder { } func borrowDecoder(r io.Reader, bufSize int) *Decoder { dec := decPool.Get().(*Decoder) - dec.called = 0 - dec.keysDone = 0 - dec.cursor = 0 - dec.err = nil dec.r = r - dec.length = 0 dec.isPooled = 0 if bufSize > 0 { dec.data = make([]byte, bufSize) @@ -62,3 +57,12 @@ func (dec *Decoder) Release() { dec.isPooled = 1 decPool.Put(dec) } + +func (dec *Decoder) reset() { + dec.called = 0 + dec.keysDone = 0 + dec.cursor = 0 + dec.err = nil + dec.length = 0 + dec.data = dec.data[:0] +} diff --git a/decode_sqlnull.go b/decode_sqlnull.go index c25549f..b376e98 100644 --- a/decode_sqlnull.go +++ b/decode_sqlnull.go @@ -7,7 +7,11 @@ func (dec *Decoder) DecodeSQLNullString(v *sql.NullString) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeSQLNullString(v) + + err := dec.decodeSQLNullString(v) + dec.reset() + + return err } func (dec *Decoder) decodeSQLNullString(v *sql.NullString) error { @@ -25,7 +29,11 @@ func (dec *Decoder) DecodeSQLNullInt64(v *sql.NullInt64) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeSQLNullInt64(v) + + err := dec.decodeSQLNullInt64(v) + dec.reset() + + return err } func (dec *Decoder) decodeSQLNullInt64(v *sql.NullInt64) error { @@ -43,7 +51,11 @@ func (dec *Decoder) DecodeSQLNullFloat64(v *sql.NullFloat64) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeSQLNullFloat64(v) + + dec.reset() + err := dec.decodeSQLNullFloat64(v) + + return err } func (dec *Decoder) decodeSQLNullFloat64(v *sql.NullFloat64) error { @@ -61,7 +73,11 @@ func (dec *Decoder) DecodeSQLNullBool(v *sql.NullBool) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeSQLNullBool(v) + + err := dec.decodeSQLNullBool(v) + dec.reset() + + return err } func (dec *Decoder) decodeSQLNullBool(v *sql.NullBool) error { diff --git a/decode_string.go b/decode_string.go index 694359c..ec63860 100644 --- a/decode_string.go +++ b/decode_string.go @@ -11,7 +11,11 @@ func (dec *Decoder) DecodeString(v *string) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeString(v) + + err := dec.decodeString(v) + dec.reset() + + return err } func (dec *Decoder) decodeString(v *string) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { diff --git a/decode_time.go b/decode_time.go index 68f906d..4a841e0 100644 --- a/decode_time.go +++ b/decode_time.go @@ -9,7 +9,11 @@ func (dec *Decoder) DecodeTime(v *time.Time, format string) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - return dec.decodeTime(v, format) + + err := dec.decodeTime(v, format) + dec.reset() + + return err } func (dec *Decoder) decodeTime(v *time.Time, format string) error { diff --git a/decode_unsafe.go b/decode_unsafe.go index 54448fb..ba9ca50 100644 --- a/decode_unsafe.go +++ b/decode_unsafe.go @@ -14,18 +14,30 @@ type decUnsafe struct{} func (u decUnsafe) UnmarshalJSONArray(data []byte, v UnmarshalerJSONArray) error { dec := borrowDecoder(nil, 0) - defer dec.Release() + + defer func() { + dec.reset() + dec.Release() + }() + dec.data = data dec.length = len(data) + _, err := dec.decodeArray(v) return err } func (u decUnsafe) UnmarshalJSONObject(data []byte, v UnmarshalerJSONObject) error { dec := borrowDecoder(nil, 0) - defer dec.Release() + + defer func() { + dec.reset() + dec.Release() + }() + dec.data = data dec.length = len(data) + _, err := dec.decodeObject(v) return err } @@ -112,9 +124,13 @@ func (u decUnsafe) Unmarshal(data []byte, v interface{}) error { default: return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, vt)) } - defer dec.Release() - if err != nil { - return err + + if err == nil { + err = dec.err } - return dec.err + + dec.reset() + dec.Release() + + return err } From 8a3cb4bf1fa82829339fb764a1a3252955607769 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 01:15:44 +0800 Subject: [PATCH 02/10] update reset implementation to cut when cursor stopped --- decode_embedded_json_test.go | 24 ++++++++++++++++++++++++ decode_pool.go | 12 +++++++++--- decode_sqlnull.go | 2 +- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/decode_embedded_json_test.go b/decode_embedded_json_test.go index a853d93..0cf19ad 100644 --- a/decode_embedded_json_test.go +++ b/decode_embedded_json_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type Request struct { @@ -178,3 +179,26 @@ func TestDecodeEmbeededJSONNil2(t *testing.T) { assert.NotNil(t, err, `err should not be nil a nil pointer is given`) assert.IsType(t, InvalidUnmarshalError(""), err, `err should not be of type InvalidUnmarshalError`) } + +func TestDecoderReuse(t *testing.T) { + r := strings.NewReader(`{"foo":"bar"}{"foo":"baz"}{"foo":"world"}`) + + dec := NewDecoder(r) + var ej EmbeddedJSON + + dec.Decode(&ej) + + require.Equal(t, `{"foo":"bar"}`, string(ej)) + + ej = ej[:0] + + dec.Decode(&ej) + + assert.Equal(t, `{"foo":"baz"}`, string(ej)) + + ej = ej[:0] + + dec.Decode(&ej) + + assert.Equal(t, `{"foo":"world"}`, string(ej)) +} diff --git a/decode_pool.go b/decode_pool.go index 0d2f878..35632b8 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -55,14 +55,20 @@ func borrowDecoder(r io.Reader, bufSize int) *Decoder { // a panic will be raised with an InvalidUsagePooledDecoderError error. func (dec *Decoder) Release() { dec.isPooled = 1 + dec.length = 0 + dec.data = dec.data[:0] decPool.Put(dec) } func (dec *Decoder) reset() { dec.called = 0 dec.keysDone = 0 - dec.cursor = 0 dec.err = nil - dec.length = 0 - dec.data = dec.data[:0] + if dec.cursor > 0 { + dec.data = dec.data[dec.cursor:] + } else { + dec.data = dec.data[:0] + } + dec.length = len(dec.data) + dec.cursor = 0 } diff --git a/decode_sqlnull.go b/decode_sqlnull.go index b376e98..63ac554 100644 --- a/decode_sqlnull.go +++ b/decode_sqlnull.go @@ -52,8 +52,8 @@ func (dec *Decoder) DecodeSQLNullFloat64(v *sql.NullFloat64) error { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - dec.reset() err := dec.decodeSQLNullFloat64(v) + dec.reset() return err } From f4c7085e75bfe6de19599aae1b9252323492eea3 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 01:24:06 +0800 Subject: [PATCH 03/10] update reset method --- decode_pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decode_pool.go b/decode_pool.go index 35632b8..4028cbd 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -64,7 +64,7 @@ func (dec *Decoder) reset() { dec.called = 0 dec.keysDone = 0 dec.err = nil - if dec.cursor > 0 { + if dec.cursor > 0 && dec.cursor < len(dec.data) { dec.data = dec.data[dec.cursor:] } else { dec.data = dec.data[:0] From effba3cbd6f35fabbfdf4fee535f40533c214657 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 01:53:20 +0800 Subject: [PATCH 04/10] add reset back to the release method --- decode_bool_test.go | 1 + decode_interface_test.go | 1 + decode_number_float_test.go | 2 ++ decode_number_int_test.go | 5 +++++ decode_number_uint_test.go | 4 ++++ decode_object_test.go | 1 + decode_pool.go | 6 ++++++ decode_sqlnull_test.go | 4 ++++ decode_stream.go | 2 ++ decode_stream_pool.go | 1 + decode_string_test.go | 1 + decode_time_test.go | 1 + 12 files changed, 29 insertions(+) diff --git a/decode_bool_test.go b/decode_bool_test.go index 99a8342..f964336 100644 --- a/decode_bool_test.go +++ b/decode_bool_test.go @@ -489,6 +489,7 @@ func TestDecoderBoolDecoderAPI(t *testing.T) { func TestDecoderBoolPoolError(t *testing.T) { v := true dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_interface_test.go b/decode_interface_test.go index 6c0e64c..8327856 100644 --- a/decode_interface_test.go +++ b/decode_interface_test.go @@ -511,6 +511,7 @@ func TestUnmarshalInterfaceError(t *testing.T) { func TestDecodeInterfacePoolError(t *testing.T) { result := interface{}(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_number_float_test.go b/decode_number_float_test.go index 4ee6dbe..b798fca 100644 --- a/decode_number_float_test.go +++ b/decode_number_float_test.go @@ -298,6 +298,7 @@ func TestDecoderFloat64(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := float64(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -931,6 +932,7 @@ func TestDecoderFloat32(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := float32(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_number_int_test.go b/decode_number_int_test.go index 909aef6..5d84630 100644 --- a/decode_number_int_test.go +++ b/decode_number_int_test.go @@ -275,6 +275,7 @@ func TestDecoderInt(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := int(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -886,6 +887,7 @@ func TestDecoderInt64(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := int64(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -1547,6 +1549,7 @@ func TestDecoderInt32(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := int32(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -2233,6 +2236,7 @@ func TestDecoderInt16(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := int16(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -2922,6 +2926,7 @@ func TestDecoderInt8(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := int8(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_number_uint_test.go b/decode_number_uint_test.go index 714bf42..d74da92 100644 --- a/decode_number_uint_test.go +++ b/decode_number_uint_test.go @@ -137,6 +137,7 @@ func TestDecoderUint64(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := uint64(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -445,6 +446,7 @@ func TestDecoderUint32(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := uint32(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -781,6 +783,7 @@ func TestDecoderUint16(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := uint16(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -1133,6 +1136,7 @@ func TestDecoderUint8(t *testing.T) { t.Run("pool-error", func(t *testing.T) { result := uint8(1) dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_object_test.go b/decode_object_test.go index fe0d741..4c3fcc6 100644 --- a/decode_object_test.go +++ b/decode_object_test.go @@ -1746,6 +1746,7 @@ func TestDecoderObjectDecoderInvalidJSONError4(t *testing.T) { func TestDecoderObjectPoolError(t *testing.T) { result := jsonDecodePartial{} dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_pool.go b/decode_pool.go index 4028cbd..a2e3490 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -56,7 +56,13 @@ func borrowDecoder(r io.Reader, bufSize int) *Decoder { func (dec *Decoder) Release() { dec.isPooled = 1 dec.length = 0 + dec.r = nil dec.data = dec.data[:0] + dec.cursor = 0 + dec.length = 0 + dec.called = 0 + dec.keysDone = 0 + dec.err = nil decPool.Put(dec) } diff --git a/decode_sqlnull_test.go b/decode_sqlnull_test.go index a8d1a26..9146191 100644 --- a/decode_sqlnull_test.go +++ b/decode_sqlnull_test.go @@ -45,6 +45,7 @@ func TestDecodeSQLNullString(t *testing.T) { "should panic because decoder is pooled", func(t *testing.T) { dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -93,6 +94,7 @@ func TestDecodeSQLNullInt64(t *testing.T) { "should panic because decoder is pooled", func(t *testing.T) { dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -141,6 +143,7 @@ func TestDecodeSQLNullFloat64(t *testing.T) { "should panic because decoder is pooled", func(t *testing.T) { dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() @@ -189,6 +192,7 @@ func TestDecodeSQLNullBool(t *testing.T) { "should panic because decoder is pooled", func(t *testing.T) { dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_stream.go b/decode_stream.go index 74beee4..617689e 100644 --- a/decode_stream.go +++ b/decode_stream.go @@ -35,11 +35,13 @@ func (dec *StreamDecoder) DecodeStream(c UnmarshalerStream) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } + if dec.r == nil { dec.err = NoReaderError("No reader given to decode stream") close(dec.done) return dec.err } + for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch dec.data[dec.cursor] { case ' ', '\n', '\t', '\r', ',': diff --git a/decode_stream_pool.go b/decode_stream_pool.go index 8e1863b..813116b 100644 --- a/decode_stream_pool.go +++ b/decode_stream_pool.go @@ -54,6 +54,7 @@ func (s stream) borrowDecoder(r io.Reader, bufSize int) *StreamDecoder { // If a decoder is used after calling Release // a panic will be raised with an InvalidUsagePooledDecoderError error. func (dec *StreamDecoder) Release() { + dec.reset() dec.isPooled = 1 streamDecPool.Put(dec) } diff --git a/decode_string_test.go b/decode_string_test.go index 4a558cb..d6f1583 100644 --- a/decode_string_test.go +++ b/decode_string_test.go @@ -668,6 +668,7 @@ func TestDecoderStringPoolError(t *testing.T) { } result := "" dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() diff --git a/decode_time_test.go b/decode_time_test.go index 8d309ed..54ece16 100644 --- a/decode_time_test.go +++ b/decode_time_test.go @@ -130,6 +130,7 @@ func TestDecoderTimePoolError(t *testing.T) { }, } dec := NewDecoder(nil) + dec.reset() dec.Release() defer func() { err := recover() From 2d9af744f306b658214c69f5d6fc31b384c4ff6d Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 02:12:08 +0800 Subject: [PATCH 05/10] avoid doing redundant stuff --- benchmarks/decoder/decoder_bench_medium_test.go | 2 +- benchmarks/decoder/decoder_bench_small_test.go | 2 +- decode_pool.go | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/benchmarks/decoder/decoder_bench_medium_test.go b/benchmarks/decoder/decoder_bench_medium_test.go index 51ef7a9..b257c57 100644 --- a/benchmarks/decoder/decoder_bench_medium_test.go +++ b/benchmarks/decoder/decoder_bench_medium_test.go @@ -36,7 +36,7 @@ func BenchmarkJSONParserDecodeObjMedium(b *testing.B) { } } -func BenchmarkEncodingJSONStructMedium(b *testing.B) { +func BenchmarkEncodingJSONDecodeObjMedium(b *testing.B) { for i := 0; i < b.N; i++ { var data = benchmarks.MediumPayload{} json.Unmarshal(benchmarks.MediumFixture, &data) diff --git a/benchmarks/decoder/decoder_bench_small_test.go b/benchmarks/decoder/decoder_bench_small_test.go index 70a5289..bb48023 100644 --- a/benchmarks/decoder/decoder_bench_small_test.go +++ b/benchmarks/decoder/decoder_bench_small_test.go @@ -12,7 +12,7 @@ import ( "github.com/mailru/easyjson" ) -func BenchmarkEncodingJSONObjSmall(b *testing.B) { +func BenchmarkEncodingJSONDecodeObjSmall(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { result := benchmarks.SmallPayload{} diff --git a/decode_pool.go b/decode_pool.go index a2e3490..e158c17 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -59,9 +59,6 @@ func (dec *Decoder) Release() { dec.r = nil dec.data = dec.data[:0] dec.cursor = 0 - dec.length = 0 - dec.called = 0 - dec.keysDone = 0 dec.err = nil decPool.Put(dec) } From 1cb50f7b876bd5abaa4ffa66c9e28cbb0c3547fe Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 02:23:10 +0800 Subject: [PATCH 06/10] remove call to reset on unmarshal methods --- decode.go | 11 ++--------- decode_pool.go | 2 ++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/decode.go b/decode.go index 2cd5130..7a02873 100644 --- a/decode.go +++ b/decode.go @@ -14,10 +14,7 @@ import ( func UnmarshalJSONArray(data []byte, v UnmarshalerJSONArray) error { dec := borrowDecoder(nil, 0) - defer func() { - dec.reset() - dec.Release() - }() + defer dec.Release() dec.data = make([]byte, len(data)) copy(dec.data, data) @@ -44,10 +41,7 @@ func UnmarshalJSONArray(data []byte, v UnmarshalerJSONArray) error { func UnmarshalJSONObject(data []byte, v UnmarshalerJSONObject) error { dec := borrowDecoder(nil, 0) - defer func() { - dec.reset() - dec.Release() - }() + defer dec.Release() dec.data = make([]byte, len(data)) copy(dec.data, data) @@ -235,7 +229,6 @@ func Unmarshal(data []byte, v interface{}) error { err = dec.err } - dec.reset() dec.Release() return err diff --git a/decode_pool.go b/decode_pool.go index e158c17..703086a 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -60,6 +60,8 @@ func (dec *Decoder) Release() { dec.data = dec.data[:0] dec.cursor = 0 dec.err = nil + dec.called = 0 + dec.keysDone = 0 decPool.Put(dec) } From cc4fb0e3fa1f1d378e5f99d6d777a144c6093630 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 16:44:31 +0800 Subject: [PATCH 07/10] reset reslice buffer at correct position --- decode_pool.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/decode_pool.go b/decode_pool.go index 703086a..e352356 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -70,10 +70,11 @@ func (dec *Decoder) reset() { dec.keysDone = 0 dec.err = nil if dec.cursor > 0 && dec.cursor < len(dec.data) { - dec.data = dec.data[dec.cursor:] + dec.data = dec.data[dec.cursor+1:] + dec.length = dec.length - dec.cursor - 1 } else { dec.data = dec.data[:0] + dec.length = 0 } - dec.length = len(dec.data) dec.cursor = 0 } From 66d8f23b433fd6b9be214842c041a23bd2979d79 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 17:43:29 +0800 Subject: [PATCH 08/10] update decode to enable reuse of decoder --- decode.go | 6 ++++-- decode_pool.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/decode.go b/decode.go index 7a02873..a38800c 100644 --- a/decode.go +++ b/decode.go @@ -326,9 +326,11 @@ func (dec *Decoder) Decode(v interface{}) error { case **bool: err = dec.decodeBoolNull(vt) case UnmarshalerJSONObject: - _, err = dec.decodeObject(vt) + dec.cursor, err = dec.decodeObject(vt) + dec.cursor++ case UnmarshalerJSONArray: - _, err = dec.decodeArray(vt) + dec.cursor, err = dec.decodeArray(vt) + dec.cursor++ case *EmbeddedJSON: err = dec.decodeEmbeddedJSON(vt) case *interface{}: diff --git a/decode_pool.go b/decode_pool.go index e352356..8c6cee8 100644 --- a/decode_pool.go +++ b/decode_pool.go @@ -70,8 +70,8 @@ func (dec *Decoder) reset() { dec.keysDone = 0 dec.err = nil if dec.cursor > 0 && dec.cursor < len(dec.data) { - dec.data = dec.data[dec.cursor+1:] - dec.length = dec.length - dec.cursor - 1 + dec.data = dec.data[dec.cursor:] + dec.length = dec.length - dec.cursor } else { dec.data = dec.data[:0] dec.length = 0 From 40491e0a5714d16e97c5ca1ede53172f12bebf2d Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 18:05:30 +0800 Subject: [PATCH 09/10] remove partial decoding on object and add example or decoder reuse --- decode.go | 2 -- decode_array.go | 5 +++- decode_object.go | 13 +++++---- decode_object_test.go | 16 ----------- examples/reuse-decoder/gojay.go | 48 +++++++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 examples/reuse-decoder/gojay.go diff --git a/decode.go b/decode.go index a38800c..964bf95 100644 --- a/decode.go +++ b/decode.go @@ -327,10 +327,8 @@ func (dec *Decoder) Decode(v interface{}) error { err = dec.decodeBoolNull(vt) case UnmarshalerJSONObject: dec.cursor, err = dec.decodeObject(vt) - dec.cursor++ case UnmarshalerJSONArray: dec.cursor, err = dec.decodeArray(vt) - dec.cursor++ case *EmbeddedJSON: err = dec.decodeEmbeddedJSON(vt) case *interface{}: diff --git a/decode_array.go b/decode_array.go index 380ca2c..b9a843a 100644 --- a/decode_array.go +++ b/decode_array.go @@ -12,7 +12,10 @@ func (dec *Decoder) DecodeArray(v UnmarshalerJSONArray) error { if dec.isPooled == 1 { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - _, err := dec.decodeArray(v) + + var err error + dec.cursor, err = dec.decodeArray(v) + dec.reset() return err diff --git a/decode_object.go b/decode_object.go index 84e6892..98392c1 100644 --- a/decode_object.go +++ b/decode_object.go @@ -15,7 +15,9 @@ func (dec *Decoder) DecodeObject(j UnmarshalerJSONObject) error { panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder")) } - _, err := dec.decodeObject(j) + var err error + dec.cursor, err = dec.decodeObject(j) + dec.reset() return err @@ -77,12 +79,9 @@ func (dec *Decoder) decodeObject(j UnmarshalerJSONObject) (int, error) { // will get to that point when keysDone is not lower than keys anymore // in that case, we make sure cursor goes to the end of object, but we skip // unmarshalling - if dec.child&1 != 0 { - end, err := dec.skipObject() - dec.cursor = end - return dec.cursor, err - } - return dec.cursor, nil + end, err := dec.skipObject() + dec.cursor = end + return dec.cursor, err case 'n': dec.cursor++ err := dec.assertNull() diff --git a/decode_object_test.go b/decode_object_test.go index 4c3fcc6..caf767b 100644 --- a/decode_object_test.go +++ b/decode_object_test.go @@ -1550,22 +1550,6 @@ func (j *jsonDecodePartial) NKeys() int { return 2 } -func TestDecodeObjectPartial(t *testing.T) { - result := jsonDecodePartial{} - dec := NewDecoder(nil) - dec.data = []byte(`{ - "test": "test", - "test2": "test", - "testArrSkip": ["test"], - "testSkipString": "test", - "testSkipNumber": 123.23 - }`) - dec.length = len(dec.data) - _, err := dec.decodeObject(&result) - assert.Nil(t, err, "err should be nil") - assert.NotEqual(t, len(dec.data), dec.cursor) -} - func TestDecoderObjectInvalidJSON(t *testing.T) { result := jsonDecodePartial{} dec := NewDecoder(nil) diff --git a/examples/reuse-decoder/gojay.go b/examples/reuse-decoder/gojay.go new file mode 100644 index 0000000..d18c15f --- /dev/null +++ b/examples/reuse-decoder/gojay.go @@ -0,0 +1,48 @@ +package main + +import ( + "bufio" + "bytes" + "io" + "runtime/debug" + "time" + + "github.com/francoispqt/gojay" +) + +func main() { + debug.SetGCPercent(5) + + b1 := make([]byte, 0, 512) + b2 := make([]byte, 0, 512) + r := bytes.NewBuffer(b1) + w := bytes.NewBuffer(b2) + + rw := bufio.NewReadWriter(bufio.NewReader(r), bufio.NewWriter(w)) + dec := gojay.NewDecoder(rw) + + go write(r) + go read(dec, rw) + + c := make(chan struct{}) + <-c +} + +func write(w io.Writer) { + for { + w.Write([]byte(`{"foo":"bar"}`)) + time.Sleep(1 * time.Millisecond) + } +} + +func read(dec *gojay.Decoder, r io.Reader) { + var msg gojay.EmbeddedJSON + for { + msg = msg[:0] + err := dec.Decode(&msg) + if err != nil { + dec = gojay.NewDecoder(r) + } + time.Sleep(1 * time.Millisecond) + } +} From 9df89bce7ea024f84eb4d44efb460bbf755e7b58 Mon Sep 17 00:00:00 2001 From: francoispqt Date: Wed, 3 Apr 2019 18:15:33 +0800 Subject: [PATCH 10/10] rename file --- examples/reuse-decoder/{gojay.go => main.go} | 4 ++++ 1 file changed, 4 insertions(+) rename examples/reuse-decoder/{gojay.go => main.go} (91%) diff --git a/examples/reuse-decoder/gojay.go b/examples/reuse-decoder/main.go similarity index 91% rename from examples/reuse-decoder/gojay.go rename to examples/reuse-decoder/main.go index d18c15f..0514bec 100644 --- a/examples/reuse-decoder/gojay.go +++ b/examples/reuse-decoder/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "bytes" + "fmt" "io" "runtime/debug" "time" @@ -42,6 +43,9 @@ func read(dec *gojay.Decoder, r io.Reader) { err := dec.Decode(&msg) if err != nil { dec = gojay.NewDecoder(r) + fmt.Println(err) + } else { + fmt.Println(string(msg)) } time.Sleep(1 * time.Millisecond) }