Skip to content

Commit d4b1023

Browse files
rjl493456442rjl493456442
committed
core/rawdb, triedb/pathdb: introduce trienode history
1 parent 0bb9fbf commit d4b1023

13 files changed

+1754
-64
lines changed

‎core/rawdb/accessors_history.go‎

Lines changed: 88 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,27 @@ func DeleteStateHistoryIndexMetadata(db ethdb.KeyValueWriter) {
4646
}
4747
}
4848

49+
// ReadTrienodeHistoryIndexMetadata retrieves the metadata of trienode history index.
50+
func ReadTrienodeHistoryIndexMetadata(db ethdb.KeyValueReader) []byte {
51+
data, _ := db.Get(headTrienodeHistoryIndexKey)
52+
return data
53+
}
54+
55+
// WriteTrienodeHistoryIndexMetadata stores the metadata of trienode history index
56+
// into database.
57+
func WriteTrienodeHistoryIndexMetadata(db ethdb.KeyValueWriter, blob []byte) {
58+
if err := db.Put(headTrienodeHistoryIndexKey, blob); err != nil {
59+
log.Crit("Failed to store the metadata of trienode history index", "err", err)
60+
}
61+
}
62+
63+
// DeleteTrienodeHistoryIndexMetadata removes the metadata of trienode history index.
64+
func DeleteTrienodeHistoryIndexMetadata(db ethdb.KeyValueWriter) {
65+
if err := db.Delete(headTrienodeHistoryIndexKey); err != nil {
66+
log.Crit("Failed to delete the metadata of trienode history index", "err", err)
67+
}
68+
}
69+
4970
// ReadAccountHistoryIndex retrieves the account history index with the provided
5071
// account address.
5172
func ReadAccountHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash) []byte {
@@ -95,6 +116,30 @@ func DeleteStorageHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash,
95116
}
96117
}
97118

119+
// ReadTrienodeHistoryIndex retrieves the trienode history index with the provided
120+
// account address and storage key hash.
121+
func ReadTrienodeHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash, path []byte) []byte {
122+
data, err := db.Get(trienodeHistoryIndexKey(addressHash, path))
123+
if err != nil || len(data) == 0 {
124+
return nil
125+
}
126+
return data
127+
}
128+
129+
// WriteTrienodeHistoryIndex writes the provided trienode history index into database.
130+
func WriteTrienodeHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, data []byte) {
131+
if err := db.Put(trienodeHistoryIndexKey(addressHash, path), data); err != nil {
132+
log.Crit("Failed to store trienode history index", "err", err)
133+
}
134+
}
135+
136+
// DeleteTrienodeHistoryIndex deletes the specified trienode index from the database.
137+
func DeleteTrienodeHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte) {
138+
if err := db.Delete(trienodeHistoryIndexKey(addressHash, path)); err != nil {
139+
log.Crit("Failed to delete trienode history index", "err", err)
140+
}
141+
}
142+
98143
// ReadAccountHistoryIndexBlock retrieves the index block with the provided
99144
// account address along with the block id.
100145
func ReadAccountHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, blockID uint32) []byte {
@@ -143,6 +188,30 @@ func DeleteStorageHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.
143188
}
144189
}
145190

191+
// ReadTrienodeHistoryIndexBlock retrieves the index block with the provided state
192+
// identifier along with the block id.
193+
func ReadTrienodeHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, path []byte, blockID uint32) []byte {
194+
data, err := db.Get(trienodeHistoryIndexBlockKey(addressHash, path, blockID))
195+
if err != nil || len(data) == 0 {
196+
return nil
197+
}
198+
return data
199+
}
200+
201+
// WriteTrienodeHistoryIndexBlock writes the provided index block into database.
202+
func WriteTrienodeHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, id uint32, data []byte) {
203+
if err := db.Put(trienodeHistoryIndexBlockKey(addressHash, path, id), data); err != nil {
204+
log.Crit("Failed to store trienode index block", "err", err)
205+
}
206+
}
207+
208+
// DeleteTrienodeHistoryIndexBlock deletes the specified index block from the database.
209+
func DeleteTrienodeHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, id uint32) {
210+
if err := db.Delete(trienodeHistoryIndexBlockKey(addressHash, path, id)); err != nil {
211+
log.Crit("Failed to delete trienode index block", "err", err)
212+
}
213+
}
214+
146215
// increaseKey increase the input key by one bit. Return nil if the entire
147216
// addition operation overflows.
148217
func increaseKey(key []byte) []byte {
@@ -155,14 +224,26 @@ func increaseKey(key []byte) []byte {
155224
return nil
156225
}
157226

158-
// DeleteStateHistoryIndex completely removes all history indexing data, including
227+
// DeleteStateHistoryIndexes completely removes all history indexing data, including
159228
// indexes for accounts and storages.
160-
//
161-
// Note, this method assumes the storage space with prefix `StateHistoryIndexPrefix`
162-
// is exclusively occupied by the history indexing data!
163-
func DeleteStateHistoryIndex(db ethdb.KeyValueRangeDeleter) {
164-
start := StateHistoryIndexPrefix
165-
limit := increaseKey(bytes.Clone(StateHistoryIndexPrefix))
229+
func DeleteStateHistoryIndexes(db ethdb.KeyValueRangeDeleter) {
230+
DeleteHistoryByRange(db, StateHistoryAccountMetadataPrefix)
231+
DeleteHistoryByRange(db, StateHistoryStorageMetadataPrefix)
232+
DeleteHistoryByRange(db, StateHistoryAccountBlockPrefix)
233+
DeleteHistoryByRange(db, StateHistoryStorageBlockPrefix)
234+
}
235+
236+
// DeleteTrienodeHistoryIndexes completely removes all trienode history indexing data.
237+
func DeleteTrienodeHistoryIndexes(db ethdb.KeyValueRangeDeleter) {
238+
DeleteHistoryByRange(db, TrienodeHistoryMetadataPrefix)
239+
DeleteHistoryByRange(db, TrienodeHistoryBlockPrefix)
240+
}
241+
242+
// DeleteHistoryByRange completely removes all database entries with the specific prefix.
243+
// Note, this method assumes the space with the given prefix is exclusively occupied!
244+
func DeleteHistoryByRange(db ethdb.KeyValueRangeDeleter, prefix []byte) {
245+
start := prefix
246+
limit := increaseKey(bytes.Clone(prefix))
166247

167248
// Try to remove the data in the range by a loop, as the leveldb
168249
// doesn't support the native range deletion.

‎core/rawdb/accessors_state.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,3 +292,76 @@ func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIn
292292
})
293293
return err
294294
}
295+
296+
// ReadTrienodeHistory retrieves the trienode history corresponding to the specified id.
297+
// Compute the position of trienode history in freezer by minus one since the id of first
298+
// trienode history starts from one(zero for initial state).
299+
func ReadTrienodeHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []byte, error) {
300+
header, err := db.Ancient(trienodeHistoryHeaderTable, id-1)
301+
if err != nil {
302+
return nil, nil, nil, err
303+
}
304+
keySection, err := db.Ancient(trienodeHistoryKeySectionTable, id-1)
305+
if err != nil {
306+
return nil, nil, nil, err
307+
}
308+
valueSection, err := db.Ancient(trienodeHistoryValueSectionTable, id-1)
309+
if err != nil {
310+
return nil, nil, nil, err
311+
}
312+
return header, keySection, valueSection, nil
313+
}
314+
315+
// ReadTrienodeHistoryHeader retrieves the header section of trienode history.
316+
func ReadTrienodeHistoryHeader(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
317+
return db.Ancient(trienodeHistoryHeaderTable, id-1)
318+
}
319+
320+
// ReadTrienodeHistoryKeySection retrieves the key section of trienode history.
321+
func ReadTrienodeHistoryKeySection(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
322+
return db.Ancient(trienodeHistoryKeySectionTable, id-1)
323+
}
324+
325+
// ReadTrienodeHistoryValueSection retrieves the value section of trienode history.
326+
func ReadTrienodeHistoryValueSection(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
327+
return db.Ancient(trienodeHistoryValueSectionTable, id-1)
328+
}
329+
330+
// ReadTrienodeHistoryList retrieves the a list of trienode history corresponding
331+
// to the specified range.
332+
// Compute the position of trienode history in freezer by minus one since the id
333+
// of first trienode history starts from one(zero for initial state).
334+
func ReadTrienodeHistoryList(db ethdb.AncientReaderOp, start uint64, count uint64) ([][]byte, [][]byte, [][]byte, error) {
335+
header, err := db.AncientRange(trienodeHistoryHeaderTable, start-1, count, 0)
336+
if err != nil {
337+
return nil, nil, nil, err
338+
}
339+
keySection, err := db.AncientRange(trienodeHistoryKeySectionTable, start-1, count, 0)
340+
if err != nil {
341+
return nil, nil, nil, err
342+
}
343+
valueSection, err := db.AncientRange(trienodeHistoryValueSectionTable, start-1, count, 0)
344+
if err != nil {
345+
return nil, nil, nil, err
346+
}
347+
if len(header) != len(keySection) || len(header) != len(valueSection) {
348+
return nil, nil, nil, errors.New("trienode history is corrupted")
349+
}
350+
return header, keySection, valueSection, nil
351+
}
352+
353+
// WriteTrienodeHistory writes the provided trienode history to database.
354+
// Compute the position of trienode history in freezer by minus one since
355+
// the id of first state history starts from one(zero for initial state).
356+
func WriteTrienodeHistory(db ethdb.AncientWriter, id uint64, header []byte, keySection []byte, valueSection []byte) error {
357+
_, err := db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
358+
if err := op.AppendRaw(trienodeHistoryHeaderTable, id-1, header); err != nil {
359+
return err
360+
}
361+
if err := op.AppendRaw(trienodeHistoryKeySectionTable, id-1, keySection); err != nil {
362+
return err
363+
}
364+
return op.AppendRaw(trienodeHistoryValueSectionTable, id-1, valueSection)
365+
})
366+
return err
367+
}

‎core/rawdb/ancient_scheme.go‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,15 +75,38 @@ var stateFreezerTableConfigs = map[string]freezerTableConfig{
7575
stateHistoryStorageData: {noSnappy: false, prunable: true},
7676
}
7777

78+
const (
79+
trienodeHistoryHeaderTable = "trienode.header"
80+
trienodeHistoryKeySectionTable = "trienode.key"
81+
trienodeHistoryValueSectionTable = "trienode.value"
82+
)
83+
84+
// trienodeFreezerTableConfigs configures the settings for tables in the trienode freezer.
85+
var trienodeFreezerTableConfigs = map[string]freezerTableConfig{
86+
trienodeHistoryHeaderTable: {noSnappy: false, prunable: true},
87+
88+
// Disable snappy compression to allow efficient partial read.
89+
trienodeHistoryKeySectionTable: {noSnappy: true, prunable: true},
90+
91+
// Disable snappy compression to allow efficient partial read.
92+
trienodeHistoryValueSectionTable: {noSnappy: true, prunable: true},
93+
}
94+
7895
// The list of identifiers of ancient stores.
7996
var (
80-
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
81-
MerkleStateFreezerName = "state" // the folder name of state history ancient store.
82-
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
97+
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
98+
MerkleStateFreezerName = "state" // the folder name of state history ancient store.
99+
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
100+
MerkleTrienodeFreezerName = "trienode" // the folder name of trienode history ancient store.
101+
VerkleTrienodeFreezerName = "trienode_verkle" // the folder name of trienode history ancient store.
83102
)
84103

85104
// freezers the collections of all builtin freezers.
86-
var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFreezerName}
105+
var freezers = []string{
106+
ChainFreezerName,
107+
MerkleStateFreezerName, VerkleStateFreezerName,
108+
MerkleTrienodeFreezerName, VerkleTrienodeFreezerName,
109+
}
87110

88111
// NewStateFreezer initializes the ancient store for state history.
89112
//
@@ -103,3 +126,22 @@ func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.Reset
103126
}
104127
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerTableConfigs)
105128
}
129+
130+
// NewTrienodeFreezer initializes the ancient store for trienode history.
131+
//
132+
// - if the empty directory is given, initializes the pure in-memory
133+
// trienode freezer (e.g. dev mode).
134+
// - if non-empty directory is given, initializes the regular file-based
135+
// trienode freezer.
136+
func NewTrienodeFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
137+
if ancientDir == "" {
138+
return NewMemoryFreezer(readOnly, trienodeFreezerTableConfigs), nil
139+
}
140+
var name string
141+
if verkle {
142+
name = filepath.Join(ancientDir, VerkleTrienodeFreezerName)
143+
} else {
144+
name = filepath.Join(ancientDir, MerkleTrienodeFreezerName)
145+
}
146+
return newResettableFreezer(name, "eth/db/trienode", readOnly, stateHistoryTableSize, trienodeFreezerTableConfigs)
147+
}

‎core/rawdb/schema.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ var (
8080
// been indexed.
8181
headStateHistoryIndexKey = []byte("LastStateHistoryIndex")
8282

83+
// headTrienodeHistoryIndexKey tracks the ID of the latest state history that has
84+
// been indexed.
85+
headTrienodeHistoryIndexKey = []byte("LastTrienodeHistoryIndex")
86+
8387
// txIndexTailKey tracks the oldest block whose transactions have been indexed.
8488
txIndexTailKey = []byte("TransactionIndexTail")
8589

@@ -125,8 +129,10 @@ var (
125129
StateHistoryIndexPrefix = []byte("m") // The global prefix of state history index data
126130
StateHistoryAccountMetadataPrefix = []byte("ma") // StateHistoryAccountMetadataPrefix + account address hash => account metadata
127131
StateHistoryStorageMetadataPrefix = []byte("ms") // StateHistoryStorageMetadataPrefix + account address hash + storage slot hash => slot metadata
132+
TrienodeHistoryMetadataPrefix = []byte("mt") // TrienodeHistoryMetadataPrefix + account address hash + trienode path => trienode metadata
128133
StateHistoryAccountBlockPrefix = []byte("mba") // StateHistoryAccountBlockPrefix + account address hash + blockID => account block
129134
StateHistoryStorageBlockPrefix = []byte("mbs") // StateHistoryStorageBlockPrefix + account address hash + storage slot hash + blockID => slot block
135+
TrienodeHistoryBlockPrefix = []byte("mbt") // TrienodeHistoryBlockPrefix + account address hash + trienode path + blockID => trienode block
130136

131137
// VerklePrefix is the database prefix for Verkle trie data, which includes:
132138
// (a) Trie nodes
@@ -395,6 +401,19 @@ func storageHistoryIndexKey(addressHash common.Hash, storageHash common.Hash) []
395401
return out
396402
}
397403

404+
// trienodeHistoryIndexKey = TrienodeHistoryMetadataPrefix + addressHash + trienode path
405+
func trienodeHistoryIndexKey(addressHash common.Hash, path []byte) []byte {
406+
totalLen := len(TrienodeHistoryMetadataPrefix) + common.HashLength + len(path)
407+
out := make([]byte, totalLen)
408+
409+
off := 0
410+
off += copy(out[off:], TrienodeHistoryMetadataPrefix)
411+
off += copy(out[off:], addressHash.Bytes())
412+
copy(out[off:], path)
413+
414+
return out
415+
}
416+
398417
// accountHistoryIndexBlockKey = StateHistoryAccountBlockPrefix + addressHash + blockID
399418
func accountHistoryIndexBlockKey(addressHash common.Hash, blockID uint32) []byte {
400419
var buf4 [4]byte
@@ -428,6 +447,23 @@ func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Has
428447
return out
429448
}
430449

450+
// trienodeHistoryIndexBlockKey = TrienodeHistoryBlockPrefix + addressHash + trienode path + blockID
451+
func trienodeHistoryIndexBlockKey(addressHash common.Hash, path []byte, blockID uint32) []byte {
452+
var buf4 [4]byte
453+
binary.BigEndian.PutUint32(buf4[:], blockID)
454+
455+
totalLen := len(TrienodeHistoryBlockPrefix) + common.HashLength + len(path) + 4
456+
out := make([]byte, totalLen)
457+
458+
off := 0
459+
off += copy(out[off:], TrienodeHistoryBlockPrefix)
460+
off += copy(out[off:], addressHash.Bytes())
461+
off += copy(out[off:], path)
462+
copy(out[off:], buf4[:])
463+
464+
return out
465+
}
466+
431467
// transitionStateKey = transitionStatusKey + hash
432468
func transitionStateKey(hash common.Hash) []byte {
433469
return append(VerkleTransitionStatePrefix, hash.Bytes()...)

‎triedb/pathdb/database.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ func (db *Database) repairHistory() error {
232232
// Purge all state history indexing data first
233233
batch := db.diskdb.NewBatch()
234234
rawdb.DeleteStateHistoryIndexMetadata(batch)
235-
rawdb.DeleteStateHistoryIndex(batch)
235+
rawdb.DeleteStateHistoryIndexes(batch)
236236
if err := batch.Write(); err != nil {
237237
log.Crit("Failed to purge state history index", "err", err)
238238
}
@@ -426,7 +426,7 @@ func (db *Database) Enable(root common.Hash) error {
426426
// Purge all state history indexing data first
427427
batch.Reset()
428428
rawdb.DeleteStateHistoryIndexMetadata(batch)
429-
rawdb.DeleteStateHistoryIndex(batch)
429+
rawdb.DeleteStateHistoryIndexes(batch)
430430
if err := batch.Write(); err != nil {
431431
return err
432432
}

0 commit comments

Comments
 (0)