feat: add toolkit for exporting and transforming missing block header fields and fix blockhash mismatch (#903)

* feat: add toolkit for exporting and transforming missing block headers fields

* feat: add missing header writer to write the missing header file with deduplicated headers

* feat: add sha256 checksum generation

* add bitmask to encapsulate logic around header information in it. also add vanity count as part of the bitmask

* feat: add optional verification of input and output file with separate csv file

* add feature to continue already existing header file

* change byte layout of deduplicated file due to too many vanities (>64) in Sepolia data

* fix some stuff

* add state root to header

* goimports

* only create 1 task at a time

* address review comments

* add optional coinbase and nonce to missing header

* add flag to run fetch from rollup relayer DB

* go mod tidy

* fix(l1 follower, rollup verifier): blockhash mismatch (#1192)

* implement missing header fields reader and manager

* chore: auto version bump [bot]

* increase download timeout

* sanitize BaseFee when executing blocks from DA

* initialize and pass missing header manager to DA syncing pipeline

* add state root to deduplicated header

* overwrite state root if given via missing header file

* fix test

* set correct links and missing header file hashes

* allow reading of previous headers by resetting file and buffer to support reset of syncing pipeline

* address review comments

* add coinbase and nonce field to missing header reader

* replace missing header reader in toolkit with actual implementation

* update sync from DA pipeline to include coinbase and nonce from missing header fields file

* update sha256 hashes for missing header fields files

* lint

* address review comments

* address review comments

* chore: auto version bump [bot]

Author: jonastheis

cmd/geth/main.go

@@ -179,6 +179,7 @@ var (
 		utils.TxGossipBroadcastDisabledFlag,
 		utils.TxGossipReceivingDisabledFlag,
 		utils.DASyncEnabledFlag,
+		utils.DAMissingHeaderFieldsBaseURLFlag,
 		utils.DABlockNativeAPIEndpointFlag,
 		utils.DABlobScanAPIEndpointFlag,
 		utils.DABeaconNodeAPIEndpointFlag,

cmd/geth/usage.go

@@ -237,6 +237,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.L1DisableMessageQueueV2Flag,
 			utils.RollupVerifyEnabledFlag,
 			utils.DASyncEnabledFlag,
+			utils.DAMissingHeaderFieldsBaseURLFlag,
 			utils.DABlobScanAPIEndpointFlag,
 			utils.DABlockNativeAPIEndpointFlag,
 			utils.DABeaconNodeAPIEndpointFlag,

cmd/utils/flags.go

@@ -908,6 +908,12 @@ var (
 		Name:  "da.sync",
 		Usage: "Enable node syncing from DA",
 	}
+	DAMissingHeaderFieldsBaseURLFlag = cli.StringFlag{
+		Name:  "da.missingheaderfields.baseurl",
+		Usage: "Base URL for fetching missing header fields for pre-EuclidV2 blocks",
+		Value: "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/",
+	}
+
 	DABlobScanAPIEndpointFlag = cli.StringFlag{
 		Name:  "da.blob.blobscan",
 		Usage: "BlobScan blob API endpoint",
@@ -1396,6 +1402,8 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 		cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name)
 	}
 
+	cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name)
+
 	if ctx.GlobalIsSet(ExternalSignerFlag.Name) {
 		cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name)
 	}

core/blockchain.go

@@ -1880,15 +1880,18 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types
 
 	header.ParentHash = parentBlock.Hash()
 
+	// sanitize base fee
+	// Note: setting the base fee to 0 will cause problems as nil != 0 when serializing the header and thus block hash will be different.
+	if header.BaseFee != nil && header.BaseFee.Cmp(common.Big0) == 0 {
+		header.BaseFee = nil
+	}
+
 	tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil)
 	receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig)
 	if err != nil {
 		return nil, NonStatTy, fmt.Errorf("error processing block: %w", err)
 	}
 
-	// TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique
-	//  This should be done with https://github.com/scroll-tech/go-ethereum/pull/913.
-
 	if sign {
 		// Prevent Engine from overriding timestamp.
 		originalTime := header.Time
@@ -1901,7 +1904,11 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types
 
 	// finalize and assemble block as fullBlock: replicates consensus.FinalizeAndAssemble()
 	header.GasUsed = gasUsed
-	header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number))
+
+	// state root might be set from partial header. If it is not set, we calculate it.
+	if header.Root == (common.Hash{}) {
+		header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number))
+	}
 
 	fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil))
 

eth/backend.go

@@ -22,6 +22,9 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"net/url"
+	"path"
+	"path/filepath"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -61,6 +64,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/rollup/ccc"
 	"github.com/scroll-tech/go-ethereum/rollup/da_syncer"
 	"github.com/scroll-tech/go-ethereum/rollup/l1"
+	"github.com/scroll-tech/go-ethereum/rollup/missing_header_fields"
 	"github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service"
 	"github.com/scroll-tech/go-ethereum/rollup/sync_service"
 	"github.com/scroll-tech/go-ethereum/rpc"
@@ -241,7 +245,12 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether
 	if config.EnableDASyncing {
 		// Do not start syncing pipeline if we are producing blocks for permissionless batches.
 		if !config.DA.ProduceBlocks {
-			eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA)
+			missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig)
+			if err != nil {
+				return nil, fmt.Errorf("cannot create missing header fields manager: %w", err)
+			}
+
+			eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA, missingHeaderFieldsManager)
 			if err != nil {
 				return nil, fmt.Errorf("cannot initialize da syncer: %w", err)
 			}
@@ -339,6 +348,22 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether
 	return eth, nil
 }
 
+func createMissingHeaderFieldsManager(stack *node.Node, chainConfig *params.ChainConfig) (*missing_header_fields.Manager, error) {
+	downloadURL, err := url.Parse(stack.Config().DAMissingHeaderFieldsBaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid DAMissingHeaderFieldsBaseURL: %w", err)
+	}
+	downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()+".bin")
+
+	expectedSHA256Checksum := chainConfig.Scroll.MissingHeaderFieldsSHA256
+	if expectedSHA256Checksum == nil {
+		return nil, fmt.Errorf("missing expected SHA256 checksum for missing header fields file in chain config")
+	}
+
+	filePath := filepath.Join(stack.Config().DataDir, fmt.Sprintf("missing-header-fields-%s-%s", chainConfig.ChainID, expectedSHA256Checksum.Hex()))
+	return missing_header_fields.NewManager(context.Background(), filePath, downloadURL.String(), *expectedSHA256Checksum), nil
+}
+
 func makeExtraData(extra []byte) []byte {
 	if len(extra) == 0 {
 		// create default extradata

node/config.go

@@ -201,6 +201,8 @@ type Config struct {
 	L1DisableMessageQueueV2 bool `toml:",omitempty"`
 	// Is daSyncingEnabled
 	DaSyncingEnabled bool `toml:",omitempty"`
+	// Base URL for missing header fields file
+	DAMissingHeaderFieldsBaseURL string `toml:",omitempty"`
 }
 
 // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into

params/config.go

@@ -30,16 +30,18 @@ import (
 
 // Genesis hashes to enforce below configs on.
 var (
-	MainnetGenesisHash        = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3")
-	RopstenGenesisHash        = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d")
-	SepoliaGenesisHash        = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9")
-	RinkebyGenesisHash        = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177")
-	GoerliGenesisHash         = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a")
-	ScrollAlphaGenesisHash    = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda")
-	ScrollSepoliaGenesisHash  = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c")
-	ScrollMainnetGenesisHash  = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80")
-	ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5")
-	ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339")
+	MainnetGenesisHash                     = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3")
+	RopstenGenesisHash                     = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d")
+	SepoliaGenesisHash                     = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9")
+	RinkebyGenesisHash                     = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177")
+	GoerliGenesisHash                      = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a")
+	ScrollAlphaGenesisHash                 = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda")
+	ScrollSepoliaGenesisHash               = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c")
+	ScrollMainnetGenesisHash               = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80")
+	ScrollSepoliaGenesisState              = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5")
+	ScrollMainnetGenesisState              = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339")
+	ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0xfa2746026ec9590e37e495cb20046e20a38fd0e7099abd2012640dddf6c88b25")
+	ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0xa02354c12ca0f918bf4768255af9ed13c137db7e56252348f304b17bb4088924")
 )
 
 func newUint64(val uint64) *uint64 { return &val }
@@ -354,7 +356,8 @@ var (
 				ScrollChainAddress:              common.HexToAddress("0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0"),
 				L2SystemConfigAddress:           common.HexToAddress("0xF444cF06A3E3724e20B35c2989d3942ea8b59124"),
 			},
-			GenesisStateRoot: &ScrollSepoliaGenesisState,
+			GenesisStateRoot:          &ScrollSepoliaGenesisState,
+			MissingHeaderFieldsSHA256: &ScrollSepoliaMissingHeaderFieldsSHA256,
 		},
 	}
 
@@ -406,7 +409,8 @@ var (
 				ScrollChainAddress:              common.HexToAddress("0xa13BAF47339d63B743e7Da8741db5456DAc1E556"),
 				L2SystemConfigAddress:           common.HexToAddress("0x331A873a2a85219863d80d248F9e2978fE88D0Ea"),
 			},
-			GenesisStateRoot: &ScrollMainnetGenesisState,
+			GenesisStateRoot:          &ScrollMainnetGenesisState,
+			MissingHeaderFieldsSHA256: &ScrollMainnetMissingHeaderFieldsSHA256,
 		},
 	}
 
@@ -710,6 +714,9 @@ type ScrollConfig struct {
 
 	// Genesis State Root for MPT clients
 	GenesisStateRoot *common.Hash `json:"genesisStateRoot,omitempty"`
+
+	// MissingHeaderFieldsSHA256 is the SHA256 hash of the missing header fields file.
+	MissingHeaderFieldsSHA256 *common.Hash `json:"missingHeaderFieldsSHA256,omitempty"`
 }
 
 // L1Config contains the l1 parameters needed to sync l1 contract events (e.g., l1 messages, commit/revert/finalize batches) in the sequencer
@@ -760,8 +767,13 @@ func (s ScrollConfig) String() string {
 		genesisStateRoot = fmt.Sprintf("%v", *s.GenesisStateRoot)
 	}
 
-	return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v}",
-		s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot)
+	missingHeaderFieldsSHA256 := "<nil>"
+	if s.MissingHeaderFieldsSHA256 != nil {
+		missingHeaderFieldsSHA256 = fmt.Sprintf("%v", *s.MissingHeaderFieldsSHA256)
+	}
+
+	return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v, missingHeaderFieldsSHA256: %v}",
+		s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot, missingHeaderFieldsSHA256)
 }
 
 // IsValidTxCount returns whether the given block's transaction count is below the limit.

params/version.go

@@ -24,7 +24,7 @@ import (
 const (
 	VersionMajor = 5         // Major version component of the current release
 	VersionMinor = 8         // Minor version component of the current release
-	VersionPatch = 65        // Patch version component of the current release
+	VersionPatch = 66        // Patch version component of the current release
 	VersionMeta  = "mainnet" // Version metadata to append to the version string
 )
 

rollup/da_syncer/block_queue.go

@@ -6,19 +6,22 @@ import (
 
 	"github.com/scroll-tech/go-ethereum/core/rawdb"
 	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/da"
+	"github.com/scroll-tech/go-ethereum/rollup/missing_header_fields"
 )
 
 // BlockQueue is a pipeline stage that reads batches from BatchQueue, extracts all da.PartialBlock from it and
 // provides them to the next stage one-by-one.
 type BlockQueue struct {
-	batchQueue *BatchQueue
-	blocks     []*da.PartialBlock
+	batchQueue                 *BatchQueue
+	blocks                     []*da.PartialBlock
+	missingHeaderFieldsManager *missing_header_fields.Manager
 }
 
-func NewBlockQueue(batchQueue *BatchQueue) *BlockQueue {
+func NewBlockQueue(batchQueue *BatchQueue, missingHeaderFieldsManager *missing_header_fields.Manager) *BlockQueue {
 	return &BlockQueue{
-		batchQueue: batchQueue,
-		blocks:     make([]*da.PartialBlock, 0),
+		batchQueue:                 batchQueue,
+		blocks:                     make([]*da.PartialBlock, 0),
+		missingHeaderFieldsManager: missingHeaderFieldsManager,
 	}
 }
 
@@ -40,7 +43,7 @@ func (bq *BlockQueue) getBlocksFromBatch(ctx context.Context) error {
 		return err
 	}
 
-	bq.blocks, err = entryWithBlocks.Blocks()
+	bq.blocks, err = entryWithBlocks.Blocks(bq.missingHeaderFieldsManager)
 	if err != nil {
 		return fmt.Errorf("failed to get blocks from entry: %w", err)
 	}

rollup/da_syncer/da/commitV0.go

@@ -13,6 +13,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/log"
 	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
 	"github.com/scroll-tech/go-ethereum/rollup/l1"
+	"github.com/scroll-tech/go-ethereum/rollup/missing_header_fields"
 )
 
 type CommitBatchDAV0 struct {
@@ -109,7 +110,7 @@ func (c *CommitBatchDAV0) CompareTo(other Entry) int {
 	return 0
 }
 
-func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) {
+func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) {
 	l1Txs, err := getL1Messages(c.db, c.parentTotalL1MessagePopped, c.skippedL1MessageBitmap, c.l1MessagesPopped)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get L1 messages for v0 batch %d: %w", c.batchIndex, err)
@@ -120,7 +121,7 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) {
 
 	curL1TxIndex := c.parentTotalL1MessagePopped
 	for _, chunk := range c.chunks {
-		for blockId, daBlock := range chunk.Blocks {
+		for blockIndex, daBlock := range chunk.Blocks {
 			// create txs
 			txs := make(types.Transactions, 0, daBlock.NumTransactions())
 			// insert l1 msgs
@@ -132,16 +133,24 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) {
 			curL1TxIndex += uint64(daBlock.NumL1Messages())
 
 			// insert l2 txs
-			txs = append(txs, chunk.Transactions[blockId]...)
+			txs = append(txs, chunk.Transactions[blockIndex]...)
+
+			difficulty, stateRoot, coinbase, nonce, extraData, err := manager.GetMissingHeaderFields(daBlock.Number())
+			if err != nil {
+				return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err)
+			}
 
 			block := NewPartialBlock(
 				&PartialHeader{
 					Number:     daBlock.Number(),
 					Time:       daBlock.Timestamp(),
 					BaseFee:    daBlock.BaseFee(),
 					GasLimit:   daBlock.GasLimit(),
-					Difficulty: 10,                             // TODO: replace with real difficulty
-					ExtraData:  []byte{1, 2, 3, 4, 5, 6, 7, 8}, // TODO: replace with real extra data
+					Difficulty: difficulty,
+					ExtraData:  extraData,
+					StateRoot:  stateRoot,
+					Coinbase:   coinbase,
+					Nonce:      nonce,
 				},
 				txs)
 			blocks = append(blocks, block)