diff --git a/encoding/codecv7.go b/encoding/codecv7.go index b3510ba..f4615dc 100644 --- a/encoding/codecv7.go +++ b/encoding/codecv7.go @@ -102,7 +102,7 @@ func (d *DACodecV7) constructBlob(batch *Batch) (*kzg4844.Blob, common.Hash, []b return nil, common.Hash{}, nil, common.Hash{}, fmt.Errorf("failed to construct blob payload: %w", err) } - compressedPayloadBytes, enableCompression, err := d.checkCompressedDataCompatibility(payloadBytes) + compressedPayloadBytes, enableCompression, err := d.checkCompressedDataCompatibility(payloadBytes, true /* checkLength */) if err != nil { return nil, common.Hash{}, nil, common.Hash{}, fmt.Errorf("failed to check batch compressed data compatibility: %w", err) } @@ -225,19 +225,22 @@ func (d *DACodecV7) DecodeTxsFromBlob(blob *kzg4844.Blob, chunks []*DAChunkRawTx // checkCompressedDataCompatibility checks the compressed data compatibility for a batch. // It constructs a blob payload, compresses the data, and checks the compressed data compatibility. -func (d *DACodecV7) checkCompressedDataCompatibility(payloadBytes []byte) ([]byte, bool, error) { +// flag checkLength indicates whether to check the length of the compressed data against the original data. +// If checkLength is true, this function returns if compression is needed based on the compressed data's length, which is used when doing batch bytes encoding. +// If checkLength is false, this function returns the result of the compatibility check, which is used when determining the chunk and batch contents. +func (d *DACodecV7) checkCompressedDataCompatibility(payloadBytes []byte, checkLength bool) ([]byte, bool, error) { compressedPayloadBytes, err := zstd.CompressScrollBatchBytes(payloadBytes) if err != nil { return nil, false, fmt.Errorf("failed to compress blob payload: %w", err) } - if err = checkCompressedDataCompatibility(compressedPayloadBytes); err != nil { + if err = checkCompressedDataCompatibilityV7(compressedPayloadBytes); err != nil { log.Warn("Compressed data compatibility check failed", "err", err, "payloadBytes", hex.EncodeToString(payloadBytes), "compressedPayloadBytes", hex.EncodeToString(compressedPayloadBytes)) return nil, false, nil } // check if compressed data is bigger or equal to the original data -> no need to compress - if len(compressedPayloadBytes) >= len(payloadBytes) { + if checkLength && len(compressedPayloadBytes) >= len(payloadBytes) { log.Warn("Compressed data is bigger or equal to the original data", "payloadBytes", hex.EncodeToString(payloadBytes), "compressedPayloadBytes", hex.EncodeToString(compressedPayloadBytes)) return nil, false, nil } @@ -246,10 +249,16 @@ func (d *DACodecV7) checkCompressedDataCompatibility(payloadBytes []byte) ([]byt } // CheckChunkCompressedDataCompatibility checks the compressed data compatibility for a batch built from a single chunk. -// Note: For DACodecV7, this function is not implemented since there is no notion of DAChunk in this version. Blobs -// contain the entire batch data, and it is up to a prover to decide the chunk sizes. -func (d *DACodecV7) CheckChunkCompressedDataCompatibility(_ *Chunk) (bool, error) { - return true, nil +func (d *DACodecV7) CheckChunkCompressedDataCompatibility(c *Chunk) (bool, error) { + // filling the needed fields for the batch used in the check + b := &Batch{ + Chunks: []*Chunk{c}, + PrevL1MessageQueueHash: c.PrevL1MessageQueueHash, + PostL1MessageQueueHash: c.PostL1MessageQueueHash, + Blocks: c.Blocks, + } + + return d.CheckBatchCompressedDataCompatibility(b) } // CheckBatchCompressedDataCompatibility checks the compressed data compatibility for a batch. @@ -267,7 +276,10 @@ func (d *DACodecV7) CheckBatchCompressedDataCompatibility(b *Batch) (bool, error return false, fmt.Errorf("failed to construct blob payload: %w", err) } - _, compatible, err := d.checkCompressedDataCompatibility(payloadBytes) + // This check is only used for sanity checks. If the check fails, it means that the compression did not work as expected. + // rollup-relayer will try popping the last chunk of the batch (or last block of the chunk when in proposing chunks) and try again to see if it works as expected. + // Since length check is used for DA and proving efficiency, it does not need to be checked here. + _, compatible, err := d.checkCompressedDataCompatibility(payloadBytes, false /* checkLength */) if err != nil { return false, fmt.Errorf("failed to check batch compressed data compatibility: %w", err) } @@ -287,7 +299,7 @@ func (d *DACodecV7) estimateL1CommitBatchSizeAndBlobSize(batch *Batch) (uint64, return 0, 0, fmt.Errorf("failed to construct blob payload: %w", err) } - compressedPayloadBytes, enableCompression, err := d.checkCompressedDataCompatibility(payloadBytes) + compressedPayloadBytes, enableCompression, err := d.checkCompressedDataCompatibility(payloadBytes, true /* checkLength */) if err != nil { return 0, 0, fmt.Errorf("failed to check batch compressed data compatibility: %w", err) } diff --git a/encoding/codecv7_test.go b/encoding/codecv7_test.go index e7a5497..4b29ea7 100644 --- a/encoding/codecv7_test.go +++ b/encoding/codecv7_test.go @@ -384,18 +384,6 @@ func TestCodecV7BatchStandardTestCasesEnableCompression(t *testing.T) { }) } - repeat := func(element byte, count int) string { - result := make([]byte, 0, count) - for i := 0; i < count; i++ { - result = append(result, element) - } - return "0x" + common.Bytes2Hex(result) - } - - // Taking into consideration compression, we allow up to 5x of max blob bytes minus 5 byte for the blob envelope header. - // We subtract 74 bytes for the blobPayloadV7 metadata. - //compressableAvailableBytes := maxEffectiveBlobBytes*5 - 5 - blobPayloadV7MinEncodedLength - maxAvailableBytesCompressable := 5*maxEffectiveBlobBytes - 5 - blobPayloadV7MinEncodedLength maxAvailableBytesIncompressable := maxEffectiveBlobBytes - 5 - blobPayloadV7MinEncodedLength // 52 bytes for each block as per daBlockV7 encoding. bytesPerBlock := 52 @@ -455,18 +443,6 @@ func TestCodecV7BatchStandardTestCasesEnableCompression(t *testing.T) { txData: []string{generateRandomData(maxAvailableBytesIncompressable/2 - bytesPerBlock*2)}, expectedBlobVersionedHash: "0x017d7f0d569464b5c74175679e5f2bc880fcf5966c3e1928c9675c942b5274f0", }, - { - name: "single block, single tx, full blob repeat data", - numBlocks: 1, - txData: []string{repeat(0x12, maxAvailableBytesCompressable-bytesPerBlock)}, - expectedBlobVersionedHash: "0x01f5d7bbfe7deb429bcbdd7347606359bca75cb93b9198e8f089b82e45f92b43", - }, - { - name: "2 blocks, single 2, full blob random data", - numBlocks: 2, - txData: []string{repeat(0x12, maxAvailableBytesCompressable/2-bytesPerBlock*2), repeat(0x13, maxAvailableBytesCompressable/2-bytesPerBlock*2)}, - expectedBlobVersionedHash: "0x01dccca3859640c50e0058fd42eaf14f942070e6497a4e2ba507b4546280a772", - }, { name: "single block, single tx, full blob random data -> error because 1 byte too big", numBlocks: 1, @@ -652,7 +628,7 @@ func TestCodecV7BatchCompressedDataCompatibilityCheck(t *testing.T) { require.NoError(t, err) // bypass batch validation checks by calling checkCompressedDataCompatibility directly - _, compatible, err := codecV7.(*DACodecV7).checkCompressedDataCompatibility([]byte{0}) + _, compatible, err := codecV7.(*DACodecV7).checkCompressedDataCompatibility([]byte{0}, true /* checkLength */) require.NoError(t, err) require.Equal(t, false, compatible) diff --git a/encoding/da.go b/encoding/da.go index ab28fab..c72c128 100644 --- a/encoding/da.go +++ b/encoding/da.go @@ -457,7 +457,51 @@ func checkCompressedDataCompatibility(data []byte) error { data = data[3+blkSize:] } - // Should we return invalid if isLast is still false? + if !isLast { + return fmt.Errorf("unexpected end before last block") + } + + return nil +} + +// Fast testing if the compressed data (v7) is compatible with our circuit +// (require specified frame header and each block is compressed) +func checkCompressedDataCompatibilityV7(data []byte) error { + if len(data) < 16 { + return fmt.Errorf("too small size (0x%x), what is it?", data) + } + + fheader := data[0] + // it is not the encoding type we expected in our zstd header + if fheader&63 != 32 { + return fmt.Errorf("unexpected header type (%x)", fheader) + } + + // skip content size + switch fheader >> 6 { + case 0: + data = data[2:] + case 1: + data = data[3:] + case 2: + data = data[5:] + case 3: + data = data[9:] + default: + panic("impossible") + } + + isLast := false + // scan each block until done + for len(data) > 3 && !isLast { + isLast = (data[0] & 1) == 1 + blkSize := (uint(data[2])*65536 + uint(data[1])*256 + uint(data[0])) >> 3 + if len(data) < 3+int(blkSize) { + return fmt.Errorf("wrong data len {%d}, expect min {%d}", len(data), 3+blkSize) + } + data = data[3+blkSize:] + } + if !isLast { return fmt.Errorf("unexpected end before last block") }