Skip to content

Commit 6ed9c95

Browse files
jonastheisjonastheis
committed
add flag to run fetch from rollup relayer DB
1 parent e02aeb3 commit 6ed9c95

File tree

4 files changed

+163
-4
lines changed

4 files changed

+163
-4
lines changed

‎rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import (
1414
"time"
1515

1616
"github.com/spf13/cobra"
17+
"gorm.io/driver/postgres"
18+
"gorm.io/gorm"
1719

1820
"github.com/scroll-tech/go-ethereum/ethclient"
1921

@@ -70,6 +72,18 @@ It produces a binary file and optionally a human readable csv file with the miss
7072
if err != nil {
7173
log.Fatalf("Error reading continue flag: %v", err)
7274
}
75+
dbDSN, err := cmd.Flags().GetString("db")
76+
if err != nil {
77+
log.Fatalf("Error reading db flag: %v", err)
78+
}
79+
80+
var db *gorm.DB
81+
if dbDSN != "" {
82+
db, err = gorm.Open(postgres.Open(dbDSN), &gorm.Config{})
83+
if err != nil {
84+
log.Fatalf("Error connecting to database: %v", err)
85+
}
86+
}
7387

7488
if continueFile != "" {
7589
fmt.Println("Continue fetching block header fields from", continueFile)
@@ -92,7 +106,7 @@ It produces a binary file and optionally a human readable csv file with the miss
92106
}
93107
}
94108

95-
runFetch(clients, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile, continueFile)
109+
runFetch(clients, db, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile, continueFile)
96110
},
97111
}
98112

@@ -107,6 +121,7 @@ func init() {
107121
fetchCmd.Flags().String("output", "headers.bin", "output file")
108122
fetchCmd.Flags().String("humanOutput", "", "additionally produce human readable csv file")
109123
fetchCmd.Flags().String("continue", "", "continue fetching block header fields from the last seen block number in the specified continue file")
124+
fetchCmd.Flags().String("db", "", "database to use instead of fetching from RPC")
110125
}
111126

112127
func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetries int) (*types.Header, error) {
@@ -118,6 +133,8 @@ func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetri
118133
header.Number.Uint64(),
119134
header.Difficulty.Uint64(),
120135
header.Root,
136+
header.Coinbase,
137+
header.Nonce,
121138
header.Extra,
122139
), nil
123140
}
@@ -132,7 +149,26 @@ func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetri
132149
return nil, fmt.Errorf("error fetching header for block %d: %v", blockNum, innerErr)
133150
}
134151

135-
func fetchHeaders(clients []*ethclient.Client, start, end uint64, headersChan chan<- *types.Header) {
152+
func fetchHeadersFromDB(db *gorm.DB, start, end uint64, headersChan chan<- *types.Header) {
153+
blockORM := types.NewL2Block(db)
154+
blocks, err := blockORM.GetL2BlocksInRange(context.Background(), start, end)
155+
if err != nil {
156+
log.Fatalf("Error fetching blocks from database: %v", err)
157+
}
158+
159+
for _, block := range blocks {
160+
headersChan <- types.NewHeader(
161+
block.Header.Number.Uint64(),
162+
block.Header.Difficulty.Uint64(),
163+
block.Header.Root,
164+
block.Header.Coinbase,
165+
block.Header.Nonce,
166+
block.Header.Extra,
167+
)
168+
}
169+
}
170+
171+
func fetchHeadersFromRPC(clients []*ethclient.Client, start, end uint64, headersChan chan<- *types.Header) {
136172
// randomize client selection to distribute load
137173
r := uint64(rand.Int())
138174

@@ -199,7 +235,7 @@ func writeHeadersToFile(outputFile string, humanReadableOutputFile string, conti
199235
fmt.Println("Finished writing headers to file, last block number:", nextHeaderNum-1)
200236
}
201237

202-
func runFetch(clients []*ethclient.Client, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string, continueFile string) {
238+
func runFetch(clients []*ethclient.Client, db *gorm.DB, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string, continueFile string) {
203239
headersChan := make(chan *types.Header, maxGoroutines*int(batchSize))
204240
tasks := make(chan task)
205241

@@ -222,12 +258,25 @@ func runFetch(clients []*ethclient.Client, startBlockNum uint64, endBlockNum uin
222258
break
223259
}
224260
log.Println("Received task", t.start, "to", t.end)
225-
fetchHeaders(clients, t.start, t.end, headersChan)
261+
262+
// use DB if dbDSN is provided, otherwise fetch from RPC
263+
if db != nil {
264+
fetchHeadersFromDB(db, t.start, t.end, headersChan)
265+
} else {
266+
fetchHeadersFromRPC(clients, t.start, t.end, headersChan)
267+
}
226268
}
227269
wgProducers.Done()
228270
}()
229271
}
230272

273+
// need to fetch block 0 from RPC
274+
if startBlockNum == 0 && db != nil {
275+
fmt.Println("Fetching headers from database... and header 0 from RPC")
276+
fetchHeadersFromRPC(clients, 0, 0, headersChan)
277+
startBlockNum = 1
278+
}
279+
231280
// create tasks/work packages for producer goroutines
232281
for start := startBlockNum; start <= endBlockNum; start += batchSize {
233282
end := start + batchSize - 1

‎rollup/missing_header_fields/export-headers-toolkit/go.mod‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,12 @@ require (
3434
github.com/huin/goupnp v1.0.2 // indirect
3535
github.com/iden3/go-iden3-crypto v0.0.16 // indirect
3636
github.com/inconshreveable/mousetrap v1.1.0 // indirect
37+
github.com/jackc/pgpassfile v1.0.0 // indirect
38+
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
39+
github.com/jackc/pgx/v5 v5.4.3 // indirect
3740
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect
41+
github.com/jinzhu/inflection v1.0.0 // indirect
42+
github.com/jinzhu/now v1.1.5 // indirect
3843
github.com/mattn/go-colorable v0.1.8 // indirect
3944
github.com/mattn/go-isatty v0.0.12 // indirect
4045
github.com/mattn/go-runewidth v0.0.15 // indirect
@@ -67,5 +72,7 @@ require (
6772
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
6873
gopkg.in/urfave/cli.v1 v1.20.0 // indirect
6974
gopkg.in/yaml.v3 v3.0.1 // indirect
75+
gorm.io/driver/postgres v1.5.7 // indirect
76+
gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde // indirect
7077
rsc.io/tmplfunc v0.0.3 // indirect
7178
)

‎rollup/missing_header_fields/export-headers-toolkit/go.sum‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,19 @@ github.com/iden3/go-iden3-crypto v0.0.16 h1:zN867xiz6HgErXVIV/6WyteGcOukE9gybYTo
9393
github.com/iden3/go-iden3-crypto v0.0.16/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBeOT/3UEhXsEsP3E=
9494
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
9595
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
96+
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
97+
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
98+
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
99+
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
100+
github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
101+
github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
96102
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
97103
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
98104
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
105+
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
106+
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
107+
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
108+
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
99109
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
100110
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
101111
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
@@ -258,5 +268,11 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
258268
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
259269
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
260270
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
271+
gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM=
272+
gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
273+
gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
274+
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
275+
gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg=
276+
gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
261277
rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
262278
rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=

‎rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package types
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"time"
8+
9+
"github.com/scroll-tech/da-codec/encoding"
10+
"github.com/scroll-tech/go-ethereum/core/types"
11+
"gorm.io/gorm"
12+
)
13+
14+
// L2Block represents a l2 block in the database.
15+
type L2Block struct {
16+
db *gorm.DB `gorm:"column:-"`
17+
18+
// block
19+
Number uint64 `json:"number" gorm:"number"`
20+
Hash string `json:"hash" gorm:"hash"`
21+
ParentHash string `json:"parent_hash" gorm:"parent_hash"`
22+
Header string `json:"header" gorm:"header"`
23+
Transactions string `json:"transactions" gorm:"transactions"`
24+
WithdrawRoot string `json:"withdraw_root" gorm:"withdraw_root"`
25+
StateRoot string `json:"state_root" gorm:"state_root"`
26+
TxNum uint32 `json:"tx_num" gorm:"tx_num"`
27+
GasUsed uint64 `json:"gas_used" gorm:"gas_used"`
28+
BlockTimestamp uint64 `json:"block_timestamp" gorm:"block_timestamp"`
29+
RowConsumption string `json:"row_consumption" gorm:"row_consumption"`
30+
31+
// chunk
32+
ChunkHash string `json:"chunk_hash" gorm:"chunk_hash;default:NULL"`
33+
34+
// metadata
35+
CreatedAt time.Time `json:"created_at" gorm:"column:created_at"`
36+
UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"`
37+
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"column:deleted_at;default:NULL"`
38+
}
39+
40+
// NewL2Block creates a new L2Block instance
41+
func NewL2Block(db *gorm.DB) *L2Block {
42+
return &L2Block{db: db}
43+
}
44+
45+
// TableName returns the name of the "l2_block" table.
46+
func (*L2Block) TableName() string {
47+
return "l2_block"
48+
}
49+
50+
// GetL2BlocksInRange retrieves the L2 blocks within the specified range (inclusive).
51+
// The range is closed, i.e., it includes both start and end block numbers.
52+
// The returned blocks are sorted in ascending order by their block number.
53+
func (o *L2Block) GetL2BlocksInRange(ctx context.Context, startBlockNumber uint64, endBlockNumber uint64) ([]*encoding.Block, error) {
54+
if startBlockNumber > endBlockNumber {
55+
return nil, fmt.Errorf("L2Block.GetL2BlocksInRange: start block number should be less than or equal to end block number, start block: %v, end block: %v", startBlockNumber, endBlockNumber)
56+
}
57+
58+
db := o.db.WithContext(ctx)
59+
db = db.Model(&L2Block{})
60+
db = db.Select("header")
61+
db = db.Where("number >= ? AND number <= ?", startBlockNumber, endBlockNumber)
62+
db = db.Order("number ASC")
63+
64+
var l2Blocks []L2Block
65+
if err := db.Find(&l2Blocks).Error; err != nil {
66+
return nil, fmt.Errorf("L2Block.GetL2BlocksInRange error: %w, start block: %v, end block: %v", err, startBlockNumber, endBlockNumber)
67+
}
68+
69+
// sanity check
70+
if uint64(len(l2Blocks)) != endBlockNumber-startBlockNumber+1 {
71+
return nil, fmt.Errorf("L2Block.GetL2BlocksInRange: unexpected number of results, expected: %v, got: %v", endBlockNumber-startBlockNumber+1, len(l2Blocks))
72+
}
73+
74+
var blocks []*encoding.Block
75+
for _, v := range l2Blocks {
76+
var block encoding.Block
77+
78+
block.Header = &types.Header{}
79+
if err := json.Unmarshal([]byte(v.Header), block.Header); err != nil {
80+
return nil, fmt.Errorf("L2Block.GetL2BlocksInRange error: %w, start block: %v, end block: %v", err, startBlockNumber, endBlockNumber)
81+
}
82+
83+
blocks = append(blocks, &block)
84+
}
85+
86+
return blocks, nil
87+
}

0 commit comments

Comments
 (0)