refactor: abstract codec versions into common interfaces (#25)

* feat: support conditional encode

* move append conditionalEncode flag after validity check

* update da-codec

* align naming

* add ConvertBlobToBlobBytes utility functions

* kept blob bytes

* rename enableEncode to enableCompress

* refactor: move some common functions to encoding (#24)

* refactor: move some common functions to encoding

* fix golint

* move symbol replace script to zstd folder

* refactor: move some util functions to public package

* fix CI

* add interfaces of codec

* add SetCompression

* move interface to encoding

* refactor

* add dablock.go

* add dachunk.go

* add dabatch.go

* move computeBatchDataHash to codecv

* fix

* add DABatchBase

* add GetCodecVersion

* add BlobVersionedHashes

* rename encoding.go to interfaces.go

* add NewDABatchWithExpectedBlobVersionedHashes

* tweak

* fix a bug

* add more logs

* add DecodeDAChunks

* add BlockRange interface

* fix

* add version check

* add Version

* remove DABatchBase

* add DABlock

* fixes

* fix

* add CodecFromVersion and CodecFromConfig

* remove GetCodecVersion

* fix typos

* make Block fields internal

* make chunk fields internal

* make batch fields internal and add some tweaks

* add JSONFromBytes

* fix a typo

* use register mode

* fix CI

* remove register mode

* add common functions

* add EstimateBlockL1CommitCalldataSize

* add dabatch interfaces

* update interface implementations

* add data hash

* fix codecv3 & codecv4 estimate gas

* fix

* fix bugs

* tweak

* add CheckChunkCompressedDataCompatibility & CheckBatchCompressedDataCompatibility

* fix

* add BlobDataProofForPointEvaluation

* add nil check in NewDAChunk

* make some util functions internal

* remove GetMaxChunksPerBatch

* fix CI

* change receiver mark from o to d

* remove SetCompression

* fix

* add GetChunkEnableCompression & GetBatchEnableCompression

* fix

* update l2geth dependency

* make types internal

* embed codecv0 <- codecv1 <- codecv2 <- codecv3 <- codecv4

* remove dablock.go dachunk.go dabatch.go

* trigger ci

* tweak

* tweaks

* add back TxsToTxsData

* fix

* fix

* fix

* fix

* fix

* fix

* tweak

* add back test block encode and test chunk encode unit tests

* replace some constants with meaningful vars

* rename daBatchV2 to daBatchV3

* add chunk hash unit tests

* add batch encode

* add batch hash unit tests

* add batch data hash & json marshal unit tests

* add calldata size unit tests

* add commit gas estimation

* add BatchSizeAndBlobSizeEstimation unit tests

* add L1MessagePopped unit tests

* add BlobEncodingAndHashing unit tests

* add blob data proof unit tests

* add TestDecodeBitmap

* add BlobCompressDecompress unit tests

* tweaks and bug fixes

* fix goimport

* add StandardTestCases

* edge cases

* fixes

* address ai's comments

* address ai's comments

* address ai's comments

* address comments

* address ai's comment

* remove some constants

* address comments

* add more logs

* address comments

* address comments

* address comments

* address comments

* add simple and nil functions unit tests

* add uncompressed case unit tests of DecodeTxsFromBlob

* add interface unit tests

* add codecv2 & codecv3 CompressedDataCompatibilityCheck unit tests

* remove mock flag

* add JSONFromBytes unit tests

* add codecv4 CompressedDataCompatibilityCheck unit tests

* add NewDABatchFromBytes unit tests

* fix golint

* tweaks

Author: colinlyguo

encoding/bitmap.go

@@ -7,8 +7,8 @@ import (
 	"github.com/scroll-tech/go-ethereum/core/types"
 )
 
-// ConstructSkippedBitmap constructs skipped L1 message bitmap of the batch.
-func ConstructSkippedBitmap(batchIndex uint64, chunks []*Chunk, totalL1MessagePoppedBefore uint64) ([]byte, uint64, error) {
+// constructSkippedBitmap constructs skipped L1 message bitmap of the batch.
+func constructSkippedBitmap(batchIndex uint64, chunks []*Chunk, totalL1MessagePoppedBefore uint64) ([]byte, uint64, error) {
 	// skipped L1 message bitmap, an array of 256-bit bitmaps
 	var skippedBitmap []*big.Int
 
@@ -54,39 +54,29 @@ func ConstructSkippedBitmap(batchIndex uint64, chunks []*Chunk, totalL1MessagePo
 		}
 	}
 
-	bitmapBytes := make([]byte, len(skippedBitmap)*32)
+	skippedL1MessageBitmap := make([]byte, len(skippedBitmap)*skippedL1MessageBitmapByteSize)
 	for ii, num := range skippedBitmap {
 		bytes := num.Bytes()
-		padding := 32 - len(bytes)
-		copy(bitmapBytes[32*ii+padding:], bytes)
+		padding := skippedL1MessageBitmapByteSize - len(bytes)
+		copy(skippedL1MessageBitmap[skippedL1MessageBitmapByteSize*ii+padding:], bytes)
 	}
 
-	return bitmapBytes, nextIndex, nil
+	return skippedL1MessageBitmap, nextIndex, nil
 }
 
-// DecodeBitmap decodes skipped L1 message bitmap of the batch from bytes to big.Int's
-func DecodeBitmap(skippedL1MessageBitmap []byte, totalL1MessagePopped int) ([]*big.Int, error) {
+// decodeBitmap decodes skipped L1 message bitmap of the batch from bytes to big.Int's.
+func decodeBitmap(skippedL1MessageBitmap []byte, totalL1MessagePopped int) ([]*big.Int, error) {
 	length := len(skippedL1MessageBitmap)
-	if length%32 != 0 {
-		return nil, fmt.Errorf("skippedL1MessageBitmap length doesn't match, skippedL1MessageBitmap length should be equal 0 modulo 32, length of skippedL1MessageBitmap: %v", length)
+	if length%skippedL1MessageBitmapByteSize != 0 {
+		return nil, fmt.Errorf("skippedL1MessageBitmap length doesn't match, skippedL1MessageBitmap length should be equal 0 modulo %v, length of skippedL1MessageBitmap: %v", skippedL1MessageBitmapByteSize, length)
 	}
 	if length*8 < totalL1MessagePopped {
 		return nil, fmt.Errorf("skippedL1MessageBitmap length is too small, skippedL1MessageBitmap length should be at least %v, length of skippedL1MessageBitmap: %v", (totalL1MessagePopped+7)/8, length)
 	}
 	var skippedBitmap []*big.Int
-	for index := 0; index < length/32; index++ {
-		bitmap := big.NewInt(0).SetBytes(skippedL1MessageBitmap[index*32 : index*32+32])
+	for index := 0; index < length/skippedL1MessageBitmapByteSize; index++ {
+		bitmap := big.NewInt(0).SetBytes(skippedL1MessageBitmap[index*skippedL1MessageBitmapByteSize : index*skippedL1MessageBitmapByteSize+skippedL1MessageBitmapByteSize])
 		skippedBitmap = append(skippedBitmap, bitmap)
 	}
 	return skippedBitmap, nil
 }
-
-// IsL1MessageSkipped checks if index is skipped in bitmap
-func IsL1MessageSkipped(skippedBitmap []*big.Int, index uint64) bool {
-	if index > uint64(len(skippedBitmap))*256 {
-		return false
-	}
-	quo := index / 256
-	rem := index % 256
-	return skippedBitmap[quo].Bit(int(rem)) != 0
-}

encoding/bitmap_test.go

@@ -0,0 +1,44 @@
+package encoding
+
+import (
+	"encoding/hex"
+	"math/big"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecodeBitmap(t *testing.T) {
+	bitmapHex := "0000000000000000000000000000000000000000000000000000001ffffffbff"
+	skippedL1MessageBitmap, err := hex.DecodeString(bitmapHex)
+	assert.NoError(t, err)
+
+	decodedBitmap, err := decodeBitmap(skippedL1MessageBitmap, 42)
+	assert.NoError(t, err)
+
+	isL1MessageSkipped := func(skippedBitmap []*big.Int, index uint64) bool {
+		if index >= uint64(len(skippedBitmap))*256 {
+			return false
+		}
+		quo := index / 256
+		rem := index % 256
+		return skippedBitmap[quo].Bit(int(rem)) == 1
+	}
+
+	assert.True(t, isL1MessageSkipped(decodedBitmap, 0))
+	assert.True(t, isL1MessageSkipped(decodedBitmap, 9))
+	assert.False(t, isL1MessageSkipped(decodedBitmap, 10))
+	assert.True(t, isL1MessageSkipped(decodedBitmap, 11))
+	assert.True(t, isL1MessageSkipped(decodedBitmap, 36))
+	assert.False(t, isL1MessageSkipped(decodedBitmap, 37))
+	assert.False(t, isL1MessageSkipped(decodedBitmap, 38))
+	assert.False(t, isL1MessageSkipped(decodedBitmap, 39))
+	assert.False(t, isL1MessageSkipped(decodedBitmap, 40))
+	assert.False(t, isL1MessageSkipped(decodedBitmap, 41))
+
+	_, err = decodeBitmap([]byte{0x00}, 8)
+	assert.Error(t, err)
+
+	_, err = decodeBitmap([]byte{0x00, 0x00, 0x00, 0x00}, 33)
+	assert.Error(t, err)
+}