codec: optimize decoding for ZeroCopy mode and use mode for benchmarks

NewDecoderBytes with ZeroCopy=true will return []byte and string that point into the input []byte.

This reduces allocation and improves performance, in both codecgen and normal mode,
showing ~ 50% reduction in allocation.

To support this, we track decByteState after during each call to DecodeBytes or DecodeStringAsByte.
Folks that care can check the value right after the call.

We also optimized json decoding, so we can return a view into the input []byte if possible,
instead of always doing a copy.

Helper functions were added to make it easy to get a string value from a []byte with optimization e.g.
zerocopy, interning, etc.

We now run benchmarks by default with ZeroCopy=true.

Author: ugorji

codec/README.md

@@ -246,7 +246,7 @@ some caveats. See Encode documentation.
 
 ```go
 const CborStreamBytes byte = 0x5f ...
-const GenVersion = 20
+const GenVersion = 21
 var SelfExt = &extFailWrapper{}
 var GoRpc goRpc
 var MsgpackSpecRpc msgpackSpecRpc

codec/bench/bench.sh

@@ -199,6 +199,7 @@ _usage() {
     printf "\t-sgx run test suite for codec; if g, use generated files; if x, do external also\n"
     printf "\t-jqp run test suite for [json, json-quick, json-profile]\n"
     printf "\t-z run tests for bench.out.txt\n"
+    printf "\t-f [pprof file] run pprof\n"
 }
 
 _main() {
@@ -211,10 +212,10 @@ _main() {
     local args=()
     local do_x="0"
     local do_g="0"
-    while getopts "dcbsjqptxklgz" flag
+    while getopts "dcbsjqptxklgzf" flag
     do
         case "$flag" in
-            d|c|b|s|j|q|p|t|x|k|l|g|z) args+=( "$flag" ) ;;
+            d|c|b|s|j|q|p|t|x|k|l|g|z|f) args+=( "$flag" ) ;;
             *) _usage; return 1 ;;
         esac
     done
@@ -237,6 +238,7 @@ _main() {
     [[ " ${args[*]} " == *"p"* ]] && _suite_very_quick_json_only_profile "$@" | _suite_trim_output
     [[ " ${args[*]} " == *"t"* ]] && _suite_tests "$@" | _suite_trim_output | _suite_tests_strip_file_line
     [[ " ${args[*]} " == *"z"* ]] && _bench_dot_out_dot_txt
+    [[ " ${args[*]} " == *"f"* ]] && ${gocmd} tool pprof bench.test ${1:-mem.out}
 
     true
     # shift $((OPTIND-1))

codec/bench/bench_test.go

@@ -78,6 +78,13 @@ func benchPreInit() {
 	// if bytesLen < approxSize {
 	// 	bytesLen = approxSize
 	// }
+
+	// use zerocopy for the benchmarks, for best performance
+	testJsonH.ZeroCopy = true
+	testCborH.ZeroCopy = true
+	testMsgpackH.ZeroCopy = true
+	testSimpleH.ZeroCopy = true
+	testBincH.ZeroCopy = true
 }
 
 func benchReinit() {

codec/bench/shared_test.go

@@ -170,9 +170,6 @@ func testHEDGet(h Handle) *testHED {
 func testReinit() {
 	testOnce = sync.Once{}
 	testHEDs = nil
-	// for _, v := range testHandles {
-	// 	v.getBasicHandle().clearInited() // so it is reinitialized next time around
-	// }
 }
 
 func testInitAll() {

codec/binc.go

@@ -702,24 +702,22 @@ func (d *bincDecDriver) decLenNumber() (v uint64) {
 	return
 }
 
-func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
+// func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
+func (d *bincDecDriver) DecodeStringAsBytes() (bs2 []byte) {
+	d.d.decByteState = decByteStateNone
 	if d.advanceNil() {
 		return
 	}
 	var slen = -1
 	switch d.vd {
 	case bincVdString, bincVdByteArray:
 		slen = d.decLen()
-		if zerocopy {
-			if d.d.bytes {
-				bs2 = d.d.decRd.rb.readx(uint(slen))
-			} else if len(bs) == 0 {
-				bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, d.d.b[:])
-			} else {
-				bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, bs)
-			}
+		if d.d.bytes {
+			d.d.decByteState = decByteStateZerocopy
+			bs2 = d.d.decRd.rb.readx(uint(slen))
 		} else {
-			bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, bs)
+			d.d.decByteState = decByteStateReuseBuf
+			bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, d.d.b[:])
 		}
 	case bincVdSymbol:
 		// zerocopy doesn't apply for symbols,
@@ -760,20 +758,21 @@ func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
 	return
 }
 
-func (d *bincDecDriver) DecodeStringAsBytes() (s []byte) {
-	return d.decStringBytes(d.d.b[:], true)
-}
-
-func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
+func (d *bincDecDriver) DecodeBytes(bs []byte) (bsOut []byte) {
+	d.d.decByteState = decByteStateNone
 	if d.advanceNil() {
 		return
 	}
 	if d.vd == bincVdArray {
-		if zerocopy && len(bs) == 0 {
+		if bs == nil {
 			bs = d.d.b[:]
+			d.d.decByteState = decByteStateReuseBuf
 		}
 		slen := d.ReadArrayStart()
-		bs = usableByteSlice(bs, slen)
+		var changed bool
+		if bs, changed = usableByteSlice(bs, slen); changed {
+			d.d.decByteState = decByteStateNone
+		}
 		for i := 0; i < slen; i++ {
 			bs[i] = uint8(chkOvf.UintV(d.DecodeUint64(), 8))
 		}
@@ -786,11 +785,13 @@ func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 		d.d.errorf("bytes - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
 	}
 	d.bdRead = false
-	if d.d.bytes && (zerocopy || d.h.ZeroCopy) {
+	if d.d.bytes && d.h.ZeroCopy {
+		d.d.decByteState = decByteStateZerocopy
 		return d.d.decRd.rb.readx(uint(clen))
 	}
-	if zerocopy && len(bs) == 0 {
+	if bs == nil {
 		bs = d.d.b[:]
+		d.d.decByteState = decByteStateReuseBuf
 	}
 	return decByteSlice(d.d.r(), clen, d.d.h.MaxInitLen, bs)
 }
@@ -802,20 +803,20 @@ func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) {
 	if d.advanceNil() {
 		return
 	}
-	realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag))
+	xbs, realxtag1, zerocopy := d.decodeExtV(ext != nil, uint8(xtag))
 	realxtag := uint64(realxtag1)
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = realxtag
-		re.Data = detachZeroCopyBytes(d.d.bytes, re.Data, xbs)
+		re.setData(xbs, zerocopy)
 	} else if ext == SelfExt {
 		d.d.sideDecode(rv, xbs)
 	} else {
 		ext.ReadExt(rv, xbs)
 	}
 }
 
-func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
+func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xbs []byte, xtag byte, zerocopy bool) {
 	if d.vd == bincVdCustomExt {
 		l := d.decLen()
 		xtag = d.d.decRd.readn1()
@@ -824,11 +825,12 @@ func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []b
 		}
 		if d.d.bytes {
 			xbs = d.d.decRd.rb.readx(uint(l))
+			zerocopy = true
 		} else {
 			xbs = decByteSlice(d.d.r(), l, d.d.h.MaxInitLen, d.d.b[:])
 		}
 	} else if d.vd == bincVdByteArray {
-		xbs = d.DecodeBytes(nil, true)
+		xbs = d.DecodeBytes(nil)
 	} else {
 		d.d.errorf("ext expects extensions or byte array - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
 	}
@@ -890,12 +892,12 @@ func (d *bincDecDriver) DecodeNaked() {
 		n.f = d.decFloat()
 	case bincVdString:
 		n.v = valueTypeString
-		n.s = d.d.string(d.DecodeStringAsBytes())
+		n.s = d.d.stringZC(d.DecodeStringAsBytes())
 	case bincVdByteArray:
-		d.d.fauxUnionReadRawBytes()
+		d.d.fauxUnionReadRawBytes(false)
 	case bincVdSymbol:
 		n.v = valueTypeSymbol
-		n.s = d.d.string(d.DecodeStringAsBytes())
+		n.s = d.d.stringZC(d.DecodeStringAsBytes())
 	case bincVdTimestamp:
 		n.v = valueTypeTime
 		tt, err := bincDecodeTime(d.d.decRd.readx(uint(d.vs)))

codec/cbor.go

@@ -572,7 +572,8 @@ func (d *cborDecDriver) ReadArrayStart() (length int) {
 	return d.decLen()
 }
 
-func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
+func (d *cborDecDriver) DecodeBytes(bs []byte) (bsOut []byte) {
+	d.d.decByteState = decByteStateNone
 	if d.advanceNil() {
 		return
 	}
@@ -582,20 +583,16 @@ func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 	if d.bd == cborBdIndefiniteBytes || d.bd == cborBdIndefiniteString {
 		d.bdRead = false
 		if bs == nil {
-			if zerocopy {
-				return d.decAppendIndefiniteBytes(d.d.b[:0])
-			}
-			return d.decAppendIndefiniteBytes(zeroByteSlice)
+			d.d.decByteState = decByteStateReuseBuf
+			return d.decAppendIndefiniteBytes(d.d.b[:0])
 		}
 		return d.decAppendIndefiniteBytes(bs[:0])
 	}
 	if d.bd == cborBdIndefiniteArray {
 		d.bdRead = false
-		if zerocopy && len(bs) == 0 {
-			bs = d.d.b[:]
-		}
 		if bs == nil {
-			bs = []byte{}
+			d.d.decByteState = decByteStateReuseBuf
+			bs = d.d.b[:0]
 		} else {
 			bs = bs[:0]
 		}
@@ -606,29 +603,35 @@ func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 	}
 	if d.bd>>5 == cborMajorArray {
 		d.bdRead = false
-		if zerocopy && len(bs) == 0 {
+		if bs == nil {
+			d.d.decByteState = decByteStateReuseBuf
 			bs = d.d.b[:]
 		}
 		slen := d.decLen()
-		bs = usableByteSlice(bs, slen)
+		var changed bool
+		if bs, changed = usableByteSlice(bs, slen); changed {
+			d.d.decByteState = decByteStateNone
+		}
 		for i := 0; i < len(bs); i++ {
 			bs[i] = uint8(chkOvf.UintV(d.DecodeUint64(), 8))
 		}
 		return bs
 	}
 	clen := d.decLen()
 	d.bdRead = false
-	if d.d.bytes && (zerocopy || d.h.ZeroCopy) {
+	if d.d.bytes && d.h.ZeroCopy {
+		d.d.decByteState = decByteStateZerocopy
 		return d.d.decRd.rb.readx(uint(clen))
 	}
-	if zerocopy && len(bs) == 0 {
+	if bs == nil {
+		d.d.decByteState = decByteStateReuseBuf
 		bs = d.d.b[:]
 	}
 	return decByteSlice(d.d.r(), clen, d.h.MaxInitLen, bs)
 }
 
 func (d *cborDecDriver) DecodeStringAsBytes() (s []byte) {
-	return d.DecodeBytes(d.d.b[:], true)
+	return d.DecodeBytes(nil)
 }
 
 func (d *cborDecDriver) DecodeTime() (t time.Time) {
@@ -704,10 +707,10 @@ func (d *cborDecDriver) DecodeNaked() {
 		n.v = valueTypeInt
 		n.i = d.DecodeInt64()
 	case cborMajorBytes:
-		d.d.fauxUnionReadRawBytes()
+		d.d.fauxUnionReadRawBytes(false)
 	case cborMajorString:
 		n.v = valueTypeString
-		n.s = d.d.string(d.DecodeStringAsBytes())
+		n.s = d.d.stringZC(d.DecodeStringAsBytes())
 	case cborMajorArray:
 		n.v = valueTypeArray
 		decodeFurther = true

codec/codec_test.go

@@ -1530,7 +1530,7 @@ func doTestMapEncodeForCanonical(t *testing.T, h Handle) {
 	if !bytes.Equal(b1t, b2t) {
 		t.Logf("Unequal bytes of length: %v vs %v", len(b1t), len(b2t))
 		if testVerbose {
-			t.Logf("Unequal bytes: %v VS %v", b1t, b2t)
+			t.Logf("Unequal bytes: \n\t%v \n\tVS \n\t%v", b1t, b2t)
 		}
 		t.FailNow()
 	}