From adb8e6c15fe410776b74e9b642f7bdeb20905f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Fri, 23 Jul 2021 15:15:53 +0200 Subject: [PATCH 01/11] Adds bestBlock hash in app state --- .../scala/io/iohk/ethereum/utils/Hex.scala | 2 + .../ethereum/ledger/BlockImporterItSpec.scala | 2 +- .../txExecTest/util/DumpChainApp.scala | 1 + .../ethereum/consensus/ConsensusImpl.scala | 6 +-- .../ethereum/db/storage/AppStateStorage.scala | 29 ++++++++++++- .../io/iohk/ethereum/domain/Blockchain.scala | 42 ++++++++++++------- .../ethereum/domain/BlockchainWriter.scala | 6 +-- .../domain/appstate/BestBlockInfo.scala | 5 +++ .../sync/EphemBlockchainTestSetup.scala | 1 + .../sync/SyncStateSchedulerSpec.scala | 2 +- .../sync/regular/RegularSyncSpec.scala | 2 +- .../validators/StdOmmersValidatorSpec.scala | 2 +- .../iohk/ethereum/domain/BlockchainSpec.scala | 10 ++--- .../jsonrpc/EthBlocksServiceSpec.scala | 24 +++++------ .../ethereum/jsonrpc/EthInfoServiceSpec.scala | 4 +- .../jsonrpc/EthProofServiceSpec.scala | 2 +- .../ethereum/jsonrpc/EthTxServiceSpec.scala | 6 +-- .../ethereum/jsonrpc/EthUserServiceSpec.scala | 10 ++--- ...pcControllerEthLegacyTransactionSpec.scala | 18 ++++---- .../jsonrpc/JsonRpcControllerEthSpec.scala | 12 +++--- .../LegacyTransactionHistoryServiceSpec.scala | 2 +- 21 files changed, 117 insertions(+), 71 deletions(-) create mode 100644 src/main/scala/io/iohk/ethereum/domain/appstate/BestBlockInfo.scala diff --git a/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala b/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala index 15dce1e589..a9e62f17f1 100644 --- a/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala +++ b/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala @@ -1,5 +1,7 @@ package io.iohk.ethereum.utils +import akka.util.ByteString + object Hex { def toHexString(bytes: Array[Byte]): String = bytes.map("%02x".format(_)).mkString diff --git a/src/it/scala/io/iohk/ethereum/ledger/BlockImporterItSpec.scala b/src/it/scala/io/iohk/ethereum/ledger/BlockImporterItSpec.scala index 33908ea701..bde35b871f 100644 --- a/src/it/scala/io/iohk/ethereum/ledger/BlockImporterItSpec.scala +++ b/src/it/scala/io/iohk/ethereum/ledger/BlockImporterItSpec.scala @@ -107,7 +107,7 @@ class BlockImporterItSpec blockchainWriter.save(oldBlock3, Nil, oldWeight3, saveAsBestBlock = true) blockchainWriter.save(oldBlock4, Nil, oldWeight4, saveAsBestBlock = true) // simulation of node restart - blockchain.saveBestKnownBlocks(blockchainReader.getBestBlockNumber() - 1) + blockchain.saveBestKnownBlocks(oldBlock3.header.hash, oldBlock3.number) blockchainWriter.save(newBlock4ParentOldBlock3, Nil, newBlock4WeightParentOldBlock3, saveAsBestBlock = true) //not reorganising anymore until oldBlock4(not part of the chain anymore), no block/ommer validation when not part of the chain, resolveBranch is returning UnknownBranch diff --git a/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala b/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala index 14138276b0..0548bb6fe1 100644 --- a/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala +++ b/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala @@ -205,4 +205,5 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain { override def getBackingMptStorage(blockNumber: BigInt): MptStorage = ??? override def getReadOnlyMptStorage(): MptStorage = ??? + } diff --git a/src/main/scala/io/iohk/ethereum/consensus/ConsensusImpl.scala b/src/main/scala/io/iohk/ethereum/consensus/ConsensusImpl.scala index f2b899936c..e04ab69e2e 100644 --- a/src/main/scala/io/iohk/ethereum/consensus/ConsensusImpl.scala +++ b/src/main/scala/io/iohk/ethereum/consensus/ConsensusImpl.scala @@ -302,9 +302,9 @@ class ConsensusImpl( case BlockData(block, _, _) if block.hasCheckpoint => block.number }.maximumOption - val bestNumber = oldBranch.last.block.header.number - blockchain.saveBestKnownBlocks(bestNumber, checkpointNumber) - executedBlocks.foreach(data => blockQueue.enqueueBlock(data.block, bestNumber)) + val bestHeader = oldBranch.last.block.header + blockchain.saveBestKnownBlocks(bestHeader.hash, bestHeader.number, checkpointNumber) + executedBlocks.foreach(data => blockQueue.enqueueBlock(data.block, bestHeader.number)) newBranch.diff(executedBlocks.map(_.block)).headOption.foreach { block => blockQueue.removeSubtree(block.header.hash) diff --git a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala index 91d7598a37..b1a759c8ad 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala @@ -1,12 +1,19 @@ package io.iohk.ethereum.db.storage -import java.math.BigInteger +import akka.util.ByteString +import boopickle.Default.Unpickle +import boopickle.Pickler +import boopickle.DefaultBasic._ +import java.math.BigInteger import scala.collection.immutable.ArraySeq - import io.iohk.ethereum.db.dataSource.DataSource import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage.AppStateStorage._ +import io.iohk.ethereum.domain.appstate.BestBlockInfo +import io.iohk.ethereum.utils.ByteUtils.{byteSequenceToBuffer, compactPickledBytes} +import io.iohk.ethereum.utils.Hex +import io.iohk.ethereum.utils.Picklers._ /** This class is used to store app state variables * Key: see AppStateStorage.Keys @@ -27,6 +34,16 @@ class AppStateStorage(val dataSource: DataSource) extends TransactionalKeyValueS def getBestBlockNumber(): BigInt = getBigInt(Keys.BestBlockNumber) + def getBestBlockData(): BestBlockInfo = + BestBlockInfo( // FIXME default value for hash ? + get(Keys.BestBlockHash).map(v => ByteString(Hex.decode(v))).getOrElse(ByteString.empty), + getBigInt(Keys.BestBlockNumber) + ) + + def putBestBlockData(b: BestBlockInfo): DataSourceBatchUpdate = + put(Keys.BestBlockNumber, b.number.toString) + .and(put(Keys.BestBlockHash, Hex.toHexString(b.hash.toArray))) + def putBestBlockNumber(bestBlockNumber: BigInt): DataSourceBatchUpdate = put(Keys.BestBlockNumber, bestBlockNumber.toString) @@ -72,9 +89,17 @@ object AppStateStorage { object Keys { val BestBlockNumber = "BestBlockNumber" + val BestBlockHash = "BestBlockHash" val FastSyncDone = "FastSyncDone" val EstimatedHighestBlock = "EstimatedHighestBlock" val SyncStartingBlock = "SyncStartingBlock" val LatestCheckpointBlockNumber = "LatestCheckpointBlockNumber" } + + implicit private val bestBlockDataPickler: Pickler[BestBlockInfo] = generatePickler[BestBlockInfo] + private val bestBlockDataSerializer = (bestBlockData: BestBlockInfo) => + compactPickledBytes(Pickle.intoBytes(bestBlockData)) + private val bestBlockDataDeserializer = + (byteSequenceToBuffer _).andThen(Unpickle[BestBlockInfo].fromBytes) + } diff --git a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala index 34efff79b1..8f3feeba05 100644 --- a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala +++ b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala @@ -1,12 +1,11 @@ package io.iohk.ethereum.domain import akka.util.ByteString - import scala.annotation.tailrec - import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage._ import io.iohk.ethereum.domain +import io.iohk.ethereum.domain.appstate.BestBlockInfo import io.iohk.ethereum.jsonrpc.ProofService.StorageProof import io.iohk.ethereum.ledger.InMemoryWorldStateProxy import io.iohk.ethereum.ledger.InMemoryWorldStateProxyStorage @@ -64,7 +63,11 @@ trait Blockchain { def removeBlock(hash: ByteString): Unit - def saveBestKnownBlocks(bestBlockNumber: BigInt, latestCheckpointNumber: Option[BigInt] = None): Unit + def saveBestKnownBlocks( + bestBlockHash: ByteString, + bestBlockNumber: BigInt, + latestCheckpointNumber: Option[BigInt] = None + ): Unit } @@ -126,20 +129,28 @@ class BlockchainImpl( def getReadOnlyMptStorage(): MptStorage = stateStorage.getReadOnlyStorage - override def saveBestKnownBlocks(bestBlockNumber: BigInt, latestCheckpointNumber: Option[BigInt] = None): Unit = + override def saveBestKnownBlocks( + bestBlockHash: ByteString, + bestBlockNumber: BigInt, + latestCheckpointNumber: Option[BigInt] = None + ): Unit = latestCheckpointNumber match { case Some(number) => - saveBestKnownBlockAndLatestCheckpointNumber(bestBlockNumber, number) + saveBestKnownBlockAndLatestCheckpointNumber(bestBlockHash, bestBlockNumber, number) case None => - saveBestKnownBlock(bestBlockNumber) + saveBestKnownBlock(bestBlockHash, bestBlockNumber) } - private def saveBestKnownBlock(bestBlockNumber: BigInt): Unit = - appStateStorage.putBestBlockNumber(bestBlockNumber).commit() + private def saveBestKnownBlock(bestBlockHash: ByteString, bestBlockNumber: BigInt): Unit = + appStateStorage.putBestBlockData(BestBlockInfo(bestBlockHash, bestBlockNumber)).commit() - private def saveBestKnownBlockAndLatestCheckpointNumber(number: BigInt, latestCheckpointNumber: BigInt): Unit = + private def saveBestKnownBlockAndLatestCheckpointNumber( + bestBlockHash: ByteString, + number: BigInt, + latestCheckpointNumber: BigInt + ): Unit = appStateStorage - .putBestBlockNumber(number) + .putBestBlockData(BestBlockInfo(bestBlockHash, number)) .and(appStateStorage.putLatestCheckpointBlockNumber(latestCheckpointNumber)) .commit() @@ -171,7 +182,8 @@ class BlockchainImpl( removeBlockNumberMapping(block.number) else blockNumberMappingStorage.emptyBatchUpdate - val newBestBlockNumber: BigInt = (bestBlockNumber - 1).max(0) + val potientialNewBestBlockNumber: BigInt = (block.number - 1).max(0) + val potientialNewBestBlockHash: ByteString = block.header.parentHash val newLatestCheckpointNumber: BigInt = if (block.hasCheckpoint && block.number == latestCheckpointNumber) { findPreviousCheckpointBlockNumber(block.number, block.number) @@ -187,8 +199,8 @@ class BlockchainImpl( into the case of having an incomplete best block and so an inconsistent db */ val bestBlockNumberUpdates = - if (appStateStorage.getBestBlockNumber() > newBestBlockNumber) - appStateStorage.putBestBlockNumber(newBestBlockNumber) + if (appStateStorage.getBestBlockNumber() > potientialNewBestBlockNumber) + appStateStorage.putBestBlockData(BestBlockInfo(potientialNewBestBlockHash, potientialNewBestBlockNumber)) else appStateStorage.emptyBatchUpdate val latestCheckpointNumberUpdates = if (appStateStorage.getLatestCheckpointBlockNumber() > newLatestCheckpointNumber) @@ -197,7 +209,7 @@ class BlockchainImpl( log.debug( "Persisting app info data into database. Persisted block number is {}. Persisted checkpoint number is {}", - newBestBlockNumber, + potientialNewBestBlockNumber, newLatestCheckpointNumber ) @@ -215,7 +227,7 @@ class BlockchainImpl( log.debug( "Removed block with hash {}. New best block number - {}, new best checkpoint block number - {}", ByteStringUtils.hash2string(blockHash), - newBestBlockNumber, + potientialNewBestBlockNumber, newLatestCheckpointNumber ) } diff --git a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala index db7d31f6d9..9b4302c205 100644 --- a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala +++ b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala @@ -1,7 +1,6 @@ package io.iohk.ethereum.domain import akka.util.ByteString - import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage.AppStateStorage import io.iohk.ethereum.db.storage.BlockBodiesStorage @@ -11,6 +10,7 @@ import io.iohk.ethereum.db.storage.ChainWeightStorage import io.iohk.ethereum.db.storage.ReceiptStorage import io.iohk.ethereum.db.storage.TransactionMappingStorage import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation +import io.iohk.ethereum.domain.appstate.BestBlockInfo import io.iohk.ethereum.utils.Logger class BlockchainWriter( @@ -31,14 +31,14 @@ class BlockchainWriter( block.header.number ) appStateStorage - .putBestBlockNumber(block.header.number) + .putBestBlockData(BestBlockInfo(block.header.hash, block.header.number)) .and(appStateStorage.putLatestCheckpointBlockNumber(block.header.number)) } else if (saveAsBestBlock) { log.debug( "New best known block number - {}", block.header.number ) - appStateStorage.putBestBlockNumber(block.header.number) + appStateStorage.putBestBlockData(BestBlockInfo(block.header.hash, block.header.number)) } else { appStateStorage.emptyBatchUpdate } diff --git a/src/main/scala/io/iohk/ethereum/domain/appstate/BestBlockInfo.scala b/src/main/scala/io/iohk/ethereum/domain/appstate/BestBlockInfo.scala new file mode 100644 index 0000000000..45a6cbfdd0 --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/domain/appstate/BestBlockInfo.scala @@ -0,0 +1,5 @@ +package io.iohk.ethereum.domain.appstate + +import akka.util.ByteString + +case class BestBlockInfo(hash: ByteString, number: BigInt) diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala index 17754acc3e..b4f6bb6f0c 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala @@ -1,5 +1,6 @@ package io.iohk.ethereum.blockchain.sync +import io.iohk.ethereum.Fixtures import io.iohk.ethereum.db.components.EphemDataSourceComponent import io.iohk.ethereum.db.components.Storages import io.iohk.ethereum.db.storage.pruning.ArchivePruning diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncStateSchedulerSpec.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncStateSchedulerSpec.scala index f1dc618ccc..a7b9eadc2a 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncStateSchedulerSpec.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncStateSchedulerSpec.scala @@ -252,7 +252,7 @@ class SyncStateSchedulerSpec buildScheduler() val header = Fixtures.Blocks.ValidBlock.header.copy(stateRoot = worldHash, number = 1) schedulerBlockchainWriter.storeBlockHeader(header).commit() - schedulerBlockchain.saveBestKnownBlocks(1) + schedulerBlockchain.saveBestKnownBlocks(header.hash, 1) var state = scheduler.initState(worldHash).get while (state.activeRequest.nonEmpty) { val (allMissingNodes1, state2) = scheduler.getAllMissingNodes(state) diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/RegularSyncSpec.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/RegularSyncSpec.scala index 0436bd93f2..2d427bb949 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/RegularSyncSpec.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/RegularSyncSpec.scala @@ -738,7 +738,7 @@ class RegularSyncSpec goToTop() val num: BigInt = 42 - blockchain.saveBestKnownBlocks(num, Some(num)) + blockchain.saveBestKnownBlocks(testBlocks.head.hash, num, Some(num)) etcPeerManager.expectMsg(GetHandshakedPeers) etcPeerManager.reply(HandshakedPeers(handshakedPeers)) diff --git a/src/test/scala/io/iohk/ethereum/consensus/pow/validators/StdOmmersValidatorSpec.scala b/src/test/scala/io/iohk/ethereum/consensus/pow/validators/StdOmmersValidatorSpec.scala index 388e18a488..6ae54d7a6b 100644 --- a/src/test/scala/io/iohk/ethereum/consensus/pow/validators/StdOmmersValidatorSpec.scala +++ b/src/test/scala/io/iohk/ethereum/consensus/pow/validators/StdOmmersValidatorSpec.scala @@ -464,7 +464,7 @@ class StdOmmersValidatorSpec extends AnyFlatSpec with Matchers with ScalaCheckPr .and(blockchainWriter.storeBlock(block95)) .and(blockchainWriter.storeBlock(block96)) .commit() - blockchain.saveBestKnownBlocks(block96.number) + blockchain.saveBestKnownBlocks(block96.hash, block96.number) } } diff --git a/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala b/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala index b932c4ac09..b0feb8a99d 100644 --- a/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala +++ b/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala @@ -44,7 +44,7 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh it should "be able to store a block and retrieve it by number" in new EphemBlockchainTestSetup { val validBlock = Fixtures.Blocks.ValidBlock.block blockchainWriter.storeBlock(validBlock).commit() - blockchain.saveBestKnownBlocks(validBlock.number) + blockchain.saveBestKnownBlocks(validBlock.hash, validBlock.number) val block = blockchainReader.getBlockByNumber(blockchainReader.getBestBranch(), validBlock.header.number) block.isDefined should ===(true) validBlock should ===(block.get) @@ -61,7 +61,7 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh blockchainWriter.save(validBlock, Seq.empty, ChainWeight(100, 100), saveAsBestBlock = true) blockchainReader.isInChain(blockchainReader.getBestBranch(), validBlock.hash) should ===(true) // simulation of node restart - blockchain.saveBestKnownBlocks(validBlock.header.number - 1) + blockchain.saveBestKnownBlocks(validBlock.header.parentHash, validBlock.header.number - 1) blockchainReader.isInChain(blockchainReader.getBestBranch(), validBlock.hash) should ===(false) } @@ -151,7 +151,7 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh val headerWithAcc = validHeader.copy(stateRoot = ByteString(mptWithAcc.getRootHash)) blockchainWriter.storeBlockHeader(headerWithAcc).commit() - blockchain.saveBestKnownBlocks(headerWithAcc.number) + blockchain.saveBestKnownBlocks(headerWithAcc.hash, headerWithAcc.number) val retrievedAccount = blockchainReader.getAccount(blockchainReader.getBestBranch(), address, headerWithAcc.number) retrievedAccount shouldEqual Some(account) @@ -171,7 +171,7 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh val headerWithAcc = validHeader.copy(stateRoot = ByteString(mptWithAcc.getRootHash)) blockchainWriter.storeBlockHeader(headerWithAcc).commit() - blockchain.saveBestKnownBlocks(headerWithAcc.number) + blockchain.saveBestKnownBlocks(headerWithAcc.hash, headerWithAcc.number) //unhappy path val wrongAddress = Address(666) @@ -200,7 +200,7 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh val headerWithAcc = Fixtures.Blocks.ValidBlock.header.copy(stateRoot = ByteString(mptWithAcc.getRootHash)) blockchainWriter.storeBlockHeader(headerWithAcc).commit() - blockchain.saveBestKnownBlocks(headerWithAcc.number) + blockchain.saveBestKnownBlocks(headerWithAcc.hash, headerWithAcc.number) val wrongAddress = Address(666) val retrievedAccountProofWrong = diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala index 38a875738c..1556194bf1 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala @@ -45,7 +45,7 @@ class EthBlocksServiceSpec "EthBlocksService" should "answer eth_blockNumber with the latest block number" in new TestSetup { val bestBlockNumber = 10 - blockchain.saveBestKnownBlocks(bestBlockNumber) + blockchain.saveBestKnownBlocks(ByteString.empty, bestBlockNumber) val response = ethBlocksService.bestBlockNumber(BestBlockNumberRequest()).runSyncUnsafe(Duration.Inf).toOption.get response.bestBlockNumber shouldEqual bestBlockNumber @@ -153,7 +153,7 @@ class EthBlocksServiceSpec .storeBlock(blockToRequest) .and(blockchainWriter.storeChainWeight(blockToRequestHash, blockWeight)) .commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.header.number) (() => blockGenerator.getPendingBlockAndState).expects().returns(None) @@ -173,7 +173,7 @@ class EthBlocksServiceSpec .storeBlock(blockToRequest) .and(blockchainWriter.storeChainWeight(blockToRequestHash, blockWeight)) .commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request = BlockByNumberRequest(BlockParam.WithNumber(blockToRequestNumber), fullTxs = true) val response = ethBlocksService.getBlockByNumber(request).runSyncUnsafe(Duration.Inf).toOption.get @@ -191,7 +191,7 @@ class EthBlocksServiceSpec it should "answer eth_getBlockByNumber with the block response correctly when it's chain weight is not in blockchain" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request = BlockByNumberRequest(BlockParam.WithNumber(blockToRequestNumber), fullTxs = true) val response = ethBlocksService.getBlockByNumber(request).runSyncUnsafe(Duration.Inf).toOption.get @@ -210,7 +210,7 @@ class EthBlocksServiceSpec .storeBlock(blockToRequest) .and(blockchainWriter.storeChainWeight(blockToRequestHash, blockWeight)) .commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request = BlockByNumberRequest(BlockParam.WithNumber(blockToRequestNumber), fullTxs = true) val response = @@ -225,7 +225,7 @@ class EthBlocksServiceSpec it should "get transaction count by block number" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val response = ethBlocksService.getBlockTransactionCountByNumber( GetBlockTransactionCountByNumberRequest(BlockParam.WithNumber(blockToRequest.header.number)) @@ -238,7 +238,7 @@ class EthBlocksServiceSpec it should "get transaction count by latest block number" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.header.number) val response = ethBlocksService.getBlockTransactionCountByNumber(GetBlockTransactionCountByNumberRequest(BlockParam.Latest)) @@ -257,7 +257,7 @@ class EthBlocksServiceSpec it should "answer eth_getUncleByBlockHashAndIndex with None when there's no uncle" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val uncleIndexToRequest = 0 val request = UncleByBlockHashAndIndexRequest(blockToRequestHash, uncleIndexToRequest) @@ -268,7 +268,7 @@ class EthBlocksServiceSpec it should "answer eth_getUncleByBlockHashAndIndex with None when there's no uncle in the requested index" in new TestSetup { blockchainWriter.storeBlock(blockToRequestWithUncles).commit() - blockchain.saveBestKnownBlocks(blockToRequestWithUncles.number) + blockchain.saveBestKnownBlocks(blockToRequestWithUncles.hash, blockToRequestWithUncles.number) val uncleIndexToRequest = 0 val request = UncleByBlockHashAndIndexRequest(blockToRequestHash, uncleIndexToRequest) @@ -361,7 +361,7 @@ class EthBlocksServiceSpec it should "answer eth_getUncleByBlockNumberAndIndex correctly when the requested index has one but there's no chain weight for it" in new TestSetup { blockchainWriter.storeBlock(blockToRequestWithUncles).commit() - blockchain.saveBestKnownBlocks(blockToRequestWithUncles.number) + blockchain.saveBestKnownBlocks(blockToRequestWithUncles.hash, blockToRequestWithUncles.number) val uncleIndexToRequest = 0 val request = UncleByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequestNumber), uncleIndexToRequest) @@ -378,7 +378,7 @@ class EthBlocksServiceSpec .storeBlock(blockToRequestWithUncles) .and(blockchainWriter.storeChainWeight(uncle.hash, uncleWeight)) .commit() - blockchain.saveBestKnownBlocks(blockToRequestWithUncles.number) + blockchain.saveBestKnownBlocks(blockToRequestWithUncles.hash, blockToRequestWithUncles.number) val uncleIndexToRequest = 0 val request = UncleByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequestNumber), uncleIndexToRequest) @@ -392,7 +392,7 @@ class EthBlocksServiceSpec it should "get uncle count by block number" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val response = ethBlocksService.getUncleCountByBlockNumber(GetUncleCountByBlockNumberRequest(BlockParam.Latest)) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthInfoServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthInfoServiceSpec.scala index 474c0df756..58ec79416e 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthInfoServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthInfoServiceSpec.scala @@ -102,7 +102,7 @@ class EthServiceSpec it should "execute call and return a value" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val worldStateProxy = InMemoryWorldStateProxy( storagesInstance.storages.evmCodeStorage, @@ -132,7 +132,7 @@ class EthServiceSpec it should "execute estimateGas and return a value" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val estimatedGas = BigInt(123) (stxLedger.binarySearchGasEstimation _).expects(*, *, *).returning(estimatedGas) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala index 776aaba6a1..07ec2d526e 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala @@ -254,7 +254,7 @@ class EthProofServiceSpec val newBlockHeader: BlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock: Block = blockToRequest.copy(header = newBlockHeader) blockchainWriter.storeBlock(newblock).commit() - blockchain.saveBestKnownBlocks(newblock.header.number) + blockchain.saveBestKnownBlocks(newblock.hash, newblock.number) val ethGetProof = new EthProofService(blockchain, blockchainReader, blockGenerator, blockchainConfig.ethCompatibleStorage) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala index 89eee0fc5b..af2a132ae0 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala @@ -174,7 +174,7 @@ class EthTxServiceSpec } it should "return average gas price" in new TestSetup { - blockchain.saveBestKnownBlocks(42) + blockchain.saveBestKnownBlocks(ByteString.empty, 42) blockchainWriter .storeBlock(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)) .commit() @@ -185,7 +185,7 @@ class EthTxServiceSpec it should "getTransactionByBlockNumberAndIndexRequest return transaction by index" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val txIndex: Int = 1 val request = GetTransactionByBlockNumberAndIndexRequest(BlockParam.Latest, txIndex) @@ -220,7 +220,7 @@ class EthTxServiceSpec it should "getRawTransactionByBlockNumberAndIndex return transaction by index" in new TestSetup { blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val txIndex: Int = 1 val request = GetTransactionByBlockNumberAndIndexRequest(BlockParam.Latest, txIndex) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala index 9e9a4fc226..f24eb41e2c 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala @@ -49,7 +49,7 @@ class EthUserServiceSpec val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) blockchainWriter.storeBlock(newblock).commit() - blockchain.saveBestKnownBlocks(newblock.header.number) + blockchain.saveBestKnownBlocks(newblock.hash, newblock.number) val response = ethUserService.getCode(GetCodeRequest(address, BlockParam.Latest)) @@ -71,7 +71,7 @@ class EthUserServiceSpec val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) blockchainWriter.storeBlock(newblock).commit() - blockchain.saveBestKnownBlocks(newblock.header.number) + blockchain.saveBestKnownBlocks(newblock.hash, newblock.number) val response = ethUserService.getBalance(GetBalanceRequest(address, BlockParam.Latest)) @@ -84,7 +84,7 @@ class EthUserServiceSpec val newBlockHeader = blockToRequest.header val newblock = blockToRequest.copy(header = newBlockHeader) blockchainWriter.storeBlock(newblock).commit() - blockchain.saveBestKnownBlocks(newblock.header.number) + blockchain.saveBestKnownBlocks(newblock.hash, newblock.header.number) val response = ethUserService.getBalance(GetBalanceRequest(address, BlockParam.Latest)) @@ -114,7 +114,7 @@ class EthUserServiceSpec val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) blockchainWriter.storeBlock(newblock).commit() - blockchain.saveBestKnownBlocks(newblock.header.number) + blockchain.saveBestKnownBlocks(newblock.hash, newblock.number) val response = ethUserService.getStorageAt(GetStorageAtRequest(address, 333, BlockParam.Latest)) response.runSyncUnsafe().map(v => UInt256(v.value)) shouldEqual Right(UInt256(123)) @@ -132,7 +132,7 @@ class EthUserServiceSpec val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) blockchainWriter.storeBlock(newblock).commit() - blockchain.saveBestKnownBlocks(newblock.header.number) + blockchain.saveBestKnownBlocks(newblock.hash, newblock.number) val response = ethUserService.getTransactionCount(GetTransactionCountRequest(address, BlockParam.Latest)) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthLegacyTransactionSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthLegacyTransactionSpec.scala index b0a9c013ac..ddf95ca71a 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthLegacyTransactionSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthLegacyTransactionSpec.scala @@ -54,7 +54,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndexToRequest = blockToRequest.body.transactionList.size / 2 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getTransactionByBlockHashAndIndex", @@ -77,7 +77,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndexToRequest = blockToRequest.body.transactionList.size / 2 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getRawTransactionByBlockHashAndIndex", @@ -137,7 +137,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndex = 1 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getTransactionByBlockNumberAndIndex", @@ -161,7 +161,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndex = 1 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getTransactionByBlockNumberAndIndex", @@ -184,7 +184,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndex = 1 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getTransactionByBlockNumberAndIndex", @@ -208,7 +208,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndex = 1 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getRawTransactionByBlockNumberAndIndex", @@ -234,7 +234,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndex = 1 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getRawTransactionByBlockNumberAndIndex", @@ -258,7 +258,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val txIndex = 1 blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getRawTransactionByBlockNumberAndIndex", @@ -336,7 +336,7 @@ class JsonRpcControllerEthLegacyTransactionSpec val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.header.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val rpcRequest = newJsonRpcRequest( "eth_getBlockTransactionCountByHash", diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala index c4d911e33c..f7be160876 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala @@ -80,7 +80,7 @@ class JsonRpcControllerEthSpec it should "handle eth_blockNumber request" in new JsonRpcControllerFixture { val bestBlockNumber = 10 - blockchain.saveBestKnownBlocks(bestBlockNumber) + blockchain.saveBestKnownBlocks(ByteString.empty, bestBlockNumber) val rpcRequest = newJsonRpcRequest("eth_blockNumber") val response = jsonRpcController.handleRequest(rpcRequest).runSyncUnsafe() @@ -177,7 +177,7 @@ class JsonRpcControllerEthSpec .storeBlock(blockToRequest) .and(blockchainWriter.storeChainWeight(blockToRequest.header.hash, blockWeight)) .commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request = newJsonRpcRequest( "eth_getBlockByNumber", @@ -199,7 +199,7 @@ class JsonRpcControllerEthSpec .storeBlock(blockToRequest) .and(blockchainWriter.storeChainWeight(blockToRequest.header.hash, blockWeight)) .commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request = newJsonRpcRequest( "eth_getBlockByNumber", @@ -221,7 +221,7 @@ class JsonRpcControllerEthSpec .storeBlock(blockToRequest) .and(blockchainWriter.storeChainWeight(blockToRequest.header.hash, blockWeight)) .commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request = newJsonRpcRequest( "eth_getBlockByNumber", @@ -265,7 +265,7 @@ class JsonRpcControllerEthSpec val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, BlockBody(Nil, Seq(uncle))) blockchainWriter.storeBlock(blockToRequest).commit() - blockchain.saveBestKnownBlocks(blockToRequest.number) + blockchain.saveBestKnownBlocks(blockToRequest.hash, blockToRequest.number) val request: JsonRpcRequest = newJsonRpcRequest( "eth_getUncleByBlockNumberAndIndex", @@ -416,7 +416,7 @@ class JsonRpcControllerEthSpec blockchainWriter .storeBlock(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)) .commit() - blockchain.saveBestKnownBlocks(42) + blockchain.saveBestKnownBlocks(ByteString.empty, 42) val request: JsonRpcRequest = newJsonRpcRequest("eth_gasPrice") diff --git a/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala b/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala index 73068ad27e..c6dd0104df 100644 --- a/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala @@ -87,7 +87,7 @@ class LegacyTransactionHistoryServiceSpec .and(blockchainWriter.storeBlock(blockWithTxs2and3)) .and(blockchainWriter.storeReceipts(blockWithTxs2and3.hash, blockTx2And3Receipts)) .commit() - blockchain.saveBestKnownBlocks(blockWithTxs2and3.number) + blockchain.saveBestKnownBlocks(blockWithTxs2and3.hash, blockWithTxs2and3.number) } response <- transactionHistoryService.getAccountTransactions(address, BigInt(3125360) to BigInt(3125370)) } yield assert(response === expectedTxs) From 2202dfbcf0d2fe687ebb36646529d2dd795beaf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Mon, 26 Jul 2021 16:22:49 +0200 Subject: [PATCH 02/11] use getBlockByHash to get the best block number instead of getBlockByNumber --- .../scala/io/iohk/ethereum/utils/Hex.scala | 2 -- .../ethereum/db/storage/AppStateStorage.scala | 20 +++++++++---------- .../io/iohk/ethereum/domain/Blockchain.scala | 1 + .../ethereum/domain/BlockchainReader.scala | 17 +++++++++++----- .../ethereum/domain/BlockchainWriter.scala | 1 + .../ethereum/consensus/ConsensusSpec.scala | 1 + .../ethereum/jsonrpc/EthTxServiceSpec.scala | 6 ++++-- .../jsonrpc/JsonRpcControllerEthSpec.scala | 8 ++++---- 8 files changed, 32 insertions(+), 24 deletions(-) diff --git a/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala b/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala index a9e62f17f1..15dce1e589 100644 --- a/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala +++ b/bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala @@ -1,7 +1,5 @@ package io.iohk.ethereum.utils -import akka.util.ByteString - object Hex { def toHexString(bytes: Array[Byte]): String = bytes.map("%02x".format(_)).mkString diff --git a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala index b1a759c8ad..86d7ee7089 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala @@ -1,17 +1,21 @@ package io.iohk.ethereum.db.storage +import java.math.BigInteger + import akka.util.ByteString + +import scala.collection.immutable.ArraySeq + import boopickle.Default.Unpickle -import boopickle.Pickler import boopickle.DefaultBasic._ +import boopickle.Pickler -import java.math.BigInteger -import scala.collection.immutable.ArraySeq import io.iohk.ethereum.db.dataSource.DataSource import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage.AppStateStorage._ import io.iohk.ethereum.domain.appstate.BestBlockInfo -import io.iohk.ethereum.utils.ByteUtils.{byteSequenceToBuffer, compactPickledBytes} +import io.iohk.ethereum.utils.ByteUtils.byteSequenceToBuffer +import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes import io.iohk.ethereum.utils.Hex import io.iohk.ethereum.utils.Picklers._ @@ -34,7 +38,7 @@ class AppStateStorage(val dataSource: DataSource) extends TransactionalKeyValueS def getBestBlockNumber(): BigInt = getBigInt(Keys.BestBlockNumber) - def getBestBlockData(): BestBlockInfo = + def getBestBlockInfo(): BestBlockInfo = BestBlockInfo( // FIXME default value for hash ? get(Keys.BestBlockHash).map(v => ByteString(Hex.decode(v))).getOrElse(ByteString.empty), getBigInt(Keys.BestBlockNumber) @@ -96,10 +100,4 @@ object AppStateStorage { val LatestCheckpointBlockNumber = "LatestCheckpointBlockNumber" } - implicit private val bestBlockDataPickler: Pickler[BestBlockInfo] = generatePickler[BestBlockInfo] - private val bestBlockDataSerializer = (bestBlockData: BestBlockInfo) => - compactPickledBytes(Pickle.intoBytes(bestBlockData)) - private val bestBlockDataDeserializer = - (byteSequenceToBuffer _).andThen(Unpickle[BestBlockInfo].fromBytes) - } diff --git a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala index 8f3feeba05..f387164528 100644 --- a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala +++ b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala @@ -2,6 +2,7 @@ package io.iohk.ethereum.domain import akka.util.ByteString import scala.annotation.tailrec + import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage._ import io.iohk.ethereum.domain diff --git a/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala b/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala index d248d2e032..c97fb943bd 100644 --- a/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala +++ b/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala @@ -1,7 +1,6 @@ package io.iohk.ethereum.domain import akka.util.ByteString - import io.iohk.ethereum.db.storage.AppStateStorage import io.iohk.ethereum.db.storage.BlockBodiesStorage import io.iohk.ethereum.db.storage.BlockHeadersStorage @@ -13,7 +12,7 @@ import io.iohk.ethereum.domain.branch.Branch import io.iohk.ethereum.domain.branch.EmptyBranch import io.iohk.ethereum.mpt.MerklePatriciaTrie import io.iohk.ethereum.mpt.MptNode -import io.iohk.ethereum.utils.Logger +import io.iohk.ethereum.utils.{Hex, Logger} class BlockchainReader( blockHeadersStorage: BlockHeadersStorage, @@ -84,9 +83,17 @@ class BlockchainReader( //returns the best known block if it's available in the storage def getBestBlock(): Option[Block] = { - val bestBlockNumber = getBestBlockNumber() - log.debug("Trying to get best block with number {}", bestBlockNumber) - getBlockByNumber(bestBlockNumber) + val bestKnownBlockinfo = appStateStorage.getBestBlockInfo() + log.debug("Trying to get best block with number {}", bestKnownBlockinfo.number) + val bestBlock = getBlockByHash(bestKnownBlockinfo.hash) + if (bestBlock.isEmpty) { + log.error( + "Best block {} (number: {}) not found in storage.", + Hex.toHexString(bestKnownBlockinfo.hash.toArray), + bestKnownBlockinfo.number + ) + } + bestBlock } def genesisHeader: BlockHeader = diff --git a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala index 9b4302c205..ded7999159 100644 --- a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala +++ b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala @@ -1,6 +1,7 @@ package io.iohk.ethereum.domain import akka.util.ByteString + import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage.AppStateStorage import io.iohk.ethereum.db.storage.BlockBodiesStorage diff --git a/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala b/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala index 763dcbf024..fa9bd4c9df 100644 --- a/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala +++ b/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala @@ -22,6 +22,7 @@ import io.iohk.ethereum.consensus.validators.BlockHeaderError.HeaderParentNotFou import io.iohk.ethereum.consensus.validators._ import io.iohk.ethereum.db.storage.MptStorage import io.iohk.ethereum.domain._ +import io.iohk.ethereum.domain.appstate.BestBlockInfo import io.iohk.ethereum.ledger.BlockData import io.iohk.ethereum.ledger.BlockExecution import io.iohk.ethereum.ledger.BlockQueue.Leaf diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala index af2a132ae0..fd60569b5c 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthTxServiceSpec.scala @@ -174,10 +174,12 @@ class EthTxServiceSpec } it should "return average gas price" in new TestSetup { - blockchain.saveBestKnownBlocks(ByteString.empty, 42) + private val block: Block = + Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body) blockchainWriter - .storeBlock(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)) + .storeBlock(block) .commit() + blockchain.saveBestKnownBlocks(block.hash, block.number) val response = ethTxService.getGetGasPrice(GetGasPriceRequest()) response.runSyncUnsafe() shouldEqual Right(GetGasPriceResponse(BigInt("20000000000"))) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala index f7be160876..9d6a70fcab 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala @@ -413,10 +413,10 @@ class JsonRpcControllerEthSpec } it should "eth_gasPrice" in new JsonRpcControllerFixture { - blockchainWriter - .storeBlock(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)) - .commit() - blockchain.saveBestKnownBlocks(ByteString.empty, 42) + private val block: Block = + Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body) + blockchainWriter.storeBlock(block).commit() + blockchain.saveBestKnownBlocks(block.hash, 42) val request: JsonRpcRequest = newJsonRpcRequest("eth_gasPrice") From 9d1311cb05636de3e9be1309fd326f6c2430d63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Mon, 26 Jul 2021 17:07:46 +0200 Subject: [PATCH 03/11] Adds a first test in BlockchainReaderSpec --- .../domain/BlockchainReaderSpec.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala diff --git a/src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala b/src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala new file mode 100644 index 0000000000..3f2c4a84ea --- /dev/null +++ b/src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala @@ -0,0 +1,19 @@ +package io.iohk.ethereum.domain + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import io.iohk.ethereum.Fixtures +import io.iohk.ethereum.blockchain.sync.EphemBlockchainTestSetup + +class BlockchainReaderSpec extends AnyFlatSpec with Matchers { + + "BlockchainReader" should "be able to get the best block after it was stored by BlockchainWriter" in new EphemBlockchainTestSetup { + val validBlock = Fixtures.Blocks.ValidBlock.block + + blockchainWriter.save(validBlock, Nil, ChainWeight.zero, true) + + blockchainReader.getBestBlock() shouldBe Some(validBlock) + } + +} From ab35f1981ea9634d85e0053c447330b7c5e14ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Tue, 27 Jul 2021 15:30:47 +0200 Subject: [PATCH 04/11] fix some warnings --- .../io/iohk/ethereum/blockchain/sync/fast/FastSync.scala | 2 +- .../io/iohk/ethereum/db/storage/AppStateStorage.scala | 7 ------- .../io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala | 2 +- .../io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala | 2 +- .../io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala | 2 +- .../transactions/LegacyTransactionHistoryServiceSpec.scala | 2 +- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/main/scala/io/iohk/ethereum/blockchain/sync/fast/FastSync.scala b/src/main/scala/io/iohk/ethereum/blockchain/sync/fast/FastSync.scala index c425ce22c1..23c412bf21 100644 --- a/src/main/scala/io/iohk/ethereum/blockchain/sync/fast/FastSync.scala +++ b/src/main/scala/io/iohk/ethereum/blockchain/sync/fast/FastSync.scala @@ -941,7 +941,7 @@ class FastSync( } def assignBlockchainWork(peerWithInfo: PeerWithInfo): Unit = { - val PeerWithInfo(peer, peerInfo) = peerWithInfo + val PeerWithInfo(peer, _) = peerWithInfo log.debug(s"Assigning blockchain work for peer [{}]", peer.id.value) if (syncState.receiptsQueue.nonEmpty) { requestReceipts(peer) diff --git a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala index 86d7ee7089..4c1f4c2b0d 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala @@ -6,18 +6,11 @@ import akka.util.ByteString import scala.collection.immutable.ArraySeq -import boopickle.Default.Unpickle -import boopickle.DefaultBasic._ -import boopickle.Pickler - import io.iohk.ethereum.db.dataSource.DataSource import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage.AppStateStorage._ import io.iohk.ethereum.domain.appstate.BestBlockInfo -import io.iohk.ethereum.utils.ByteUtils.byteSequenceToBuffer -import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes import io.iohk.ethereum.utils.Hex -import io.iohk.ethereum.utils.Picklers._ /** This class is used to store app state variables * Key: see AppStateStorage.Keys diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala index 1556194bf1..9d73335ce2 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthBlocksServiceSpec.scala @@ -412,7 +412,7 @@ class EthBlocksServiceSpec ) } - class TestSetup(implicit system: ActorSystem) extends MockFactory with EphemBlockchainTestSetup { + class TestSetup() extends MockFactory with EphemBlockchainTestSetup { val blockGenerator: PoWBlockGenerator = mock[PoWBlockGenerator] val appStateStorage: AppStateStorage = mock[AppStateStorage] diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala index 07ec2d526e..0300815020 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthProofServiceSpec.scala @@ -214,7 +214,7 @@ class EthProofServiceSpec ) } - class TestSetup(implicit system: ActorSystem) extends MockFactory with EphemBlockchainTestSetup with ApisBuilder { + class TestSetup() extends MockFactory with EphemBlockchainTestSetup with ApisBuilder { val blockGenerator: PoWBlockGenerator = mock[PoWBlockGenerator] val address: Address = Address(ByteString(Hex.decode("abbb6bebfa05aa13e908eaa492bd7a8343760477"))) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala index f24eb41e2c..f5d2144549 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthUserServiceSpec.scala @@ -139,7 +139,7 @@ class EthUserServiceSpec response.runSyncUnsafe() shouldEqual Right(GetTransactionCountResponse(BigInt(999))) } - class TestSetup(implicit system: ActorSystem) extends MockFactory with EphemBlockchainTestSetup { + class TestSetup() extends MockFactory with EphemBlockchainTestSetup { lazy val ethUserService = new EthUserService( blockchain, blockchainReader, diff --git a/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala b/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala index c6dd0104df..28e5efad93 100644 --- a/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/transactions/LegacyTransactionHistoryServiceSpec.scala @@ -160,7 +160,7 @@ class LegacyTransactionHistoryServiceSpec ) def makeReceipts(block: Block): Seq[Receipt] = - block.body.transactionList.map(tx => Receipt(HashOutcome(block.hash), BigInt(21000), ByteString("foo"), Nil)) + block.body.transactionList.map(_ => Receipt(HashOutcome(block.hash), BigInt(21000), ByteString("foo"), Nil)) for { _ <- Task { From 07716513e70df41d0e0b281fd24a33ae49f38467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Mon, 2 Aug 2021 11:37:11 +0200 Subject: [PATCH 05/11] fix typo --- .../scala/io/iohk/ethereum/domain/Blockchain.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala index f387164528..5f27756b0f 100644 --- a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala +++ b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala @@ -1,6 +1,7 @@ package io.iohk.ethereum.domain import akka.util.ByteString + import scala.annotation.tailrec import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate @@ -183,8 +184,8 @@ class BlockchainImpl( removeBlockNumberMapping(block.number) else blockNumberMappingStorage.emptyBatchUpdate - val potientialNewBestBlockNumber: BigInt = (block.number - 1).max(0) - val potientialNewBestBlockHash: ByteString = block.header.parentHash + val potentialNewBestBlockNumber: BigInt = (block.number - 1).max(0) + val potentialNewBestBlockHash: ByteString = block.header.parentHash val newLatestCheckpointNumber: BigInt = if (block.hasCheckpoint && block.number == latestCheckpointNumber) { findPreviousCheckpointBlockNumber(block.number, block.number) @@ -200,8 +201,8 @@ class BlockchainImpl( into the case of having an incomplete best block and so an inconsistent db */ val bestBlockNumberUpdates = - if (appStateStorage.getBestBlockNumber() > potientialNewBestBlockNumber) - appStateStorage.putBestBlockData(BestBlockInfo(potientialNewBestBlockHash, potientialNewBestBlockNumber)) + if (appStateStorage.getBestBlockNumber() > potentialNewBestBlockNumber) + appStateStorage.putBestBlockData(BestBlockInfo(potentialNewBestBlockHash, potentialNewBestBlockNumber)) else appStateStorage.emptyBatchUpdate val latestCheckpointNumberUpdates = if (appStateStorage.getLatestCheckpointBlockNumber() > newLatestCheckpointNumber) @@ -210,7 +211,7 @@ class BlockchainImpl( log.debug( "Persisting app info data into database. Persisted block number is {}. Persisted checkpoint number is {}", - potientialNewBestBlockNumber, + potentialNewBestBlockNumber, newLatestCheckpointNumber ) @@ -228,7 +229,7 @@ class BlockchainImpl( log.debug( "Removed block with hash {}. New best block number - {}, new best checkpoint block number - {}", ByteStringUtils.hash2string(blockHash), - potientialNewBestBlockNumber, + potentialNewBestBlockNumber, newLatestCheckpointNumber ) } From 30c542c444f9d4a35e0488e5d6a2b023bcb61570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Wed, 4 Aug 2021 09:45:14 +0200 Subject: [PATCH 06/11] review fixes rename putBestBlockData to putBestBlockInfo avoid using magic number for block number in test --- .../io/iohk/ethereum/txExecTest/util/DumpChainApp.scala | 6 +++++- .../scala/io/iohk/ethereum/db/storage/AppStateStorage.scala | 2 +- src/main/scala/io/iohk/ethereum/domain/Blockchain.scala | 6 +++--- .../scala/io/iohk/ethereum/domain/BlockchainWriter.scala | 4 ++-- .../io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala b/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala index 0548bb6fe1..cf91caefef 100644 --- a/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala +++ b/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala @@ -196,7 +196,11 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain { def getBestBlockNumber(): BigInt = ??? - def saveBestKnownBlocks(bestBlockNumber: BigInt, latestCheckpointNumber: Option[BigInt] = None): Unit = ??? + override def saveBestKnownBlocks( + bestBlockhash: ByteString, + bestBlockNumber: BigInt, + latestCheckpointNumber: Option[BigInt] = None + ): Unit = ??? def getBestBlock(): Option[Block] = ??? diff --git a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala index 4c1f4c2b0d..7d7e71a1e4 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala @@ -37,7 +37,7 @@ class AppStateStorage(val dataSource: DataSource) extends TransactionalKeyValueS getBigInt(Keys.BestBlockNumber) ) - def putBestBlockData(b: BestBlockInfo): DataSourceBatchUpdate = + def putBestBlockInfo(b: BestBlockInfo): DataSourceBatchUpdate = put(Keys.BestBlockNumber, b.number.toString) .and(put(Keys.BestBlockHash, Hex.toHexString(b.hash.toArray))) diff --git a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala index 5f27756b0f..3147ad1c7d 100644 --- a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala +++ b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala @@ -144,7 +144,7 @@ class BlockchainImpl( } private def saveBestKnownBlock(bestBlockHash: ByteString, bestBlockNumber: BigInt): Unit = - appStateStorage.putBestBlockData(BestBlockInfo(bestBlockHash, bestBlockNumber)).commit() + appStateStorage.putBestBlockInfo(BestBlockInfo(bestBlockHash, bestBlockNumber)).commit() private def saveBestKnownBlockAndLatestCheckpointNumber( bestBlockHash: ByteString, @@ -152,7 +152,7 @@ class BlockchainImpl( latestCheckpointNumber: BigInt ): Unit = appStateStorage - .putBestBlockData(BestBlockInfo(bestBlockHash, number)) + .putBestBlockInfo(BestBlockInfo(bestBlockHash, number)) .and(appStateStorage.putLatestCheckpointBlockNumber(latestCheckpointNumber)) .commit() @@ -202,7 +202,7 @@ class BlockchainImpl( */ val bestBlockNumberUpdates = if (appStateStorage.getBestBlockNumber() > potentialNewBestBlockNumber) - appStateStorage.putBestBlockData(BestBlockInfo(potentialNewBestBlockHash, potentialNewBestBlockNumber)) + appStateStorage.putBestBlockInfo(BestBlockInfo(potentialNewBestBlockHash, potentialNewBestBlockNumber)) else appStateStorage.emptyBatchUpdate val latestCheckpointNumberUpdates = if (appStateStorage.getLatestCheckpointBlockNumber() > newLatestCheckpointNumber) diff --git a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala index ded7999159..615992b05b 100644 --- a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala +++ b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala @@ -32,14 +32,14 @@ class BlockchainWriter( block.header.number ) appStateStorage - .putBestBlockData(BestBlockInfo(block.header.hash, block.header.number)) + .putBestBlockInfo(BestBlockInfo(block.header.hash, block.header.number)) .and(appStateStorage.putLatestCheckpointBlockNumber(block.header.number)) } else if (saveAsBestBlock) { log.debug( "New best known block number - {}", block.header.number ) - appStateStorage.putBestBlockData(BestBlockInfo(block.header.hash, block.header.number)) + appStateStorage.putBestBlockInfo(BestBlockInfo(block.header.hash, block.header.number)) } else { appStateStorage.emptyBatchUpdate } diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala index 9d6a70fcab..6f3d63eaef 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerEthSpec.scala @@ -416,7 +416,7 @@ class JsonRpcControllerEthSpec private val block: Block = Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body) blockchainWriter.storeBlock(block).commit() - blockchain.saveBestKnownBlocks(block.hash, 42) + blockchain.saveBestKnownBlocks(block.hash, block.number) val request: JsonRpcRequest = newJsonRpcRequest("eth_gasPrice") From 78f2869a2249d073062a772d2c326b200414e7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Mon, 9 Aug 2021 15:00:25 +0200 Subject: [PATCH 07/11] use property test to test BlockchainReader --- .../ethereum/db/storage/AppStateStorage.scala | 2 +- .../ethereum/domain/BlockchainReaderSpec.scala | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala index 7d7e71a1e4..119ebb9da3 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala @@ -32,7 +32,7 @@ class AppStateStorage(val dataSource: DataSource) extends TransactionalKeyValueS getBigInt(Keys.BestBlockNumber) def getBestBlockInfo(): BestBlockInfo = - BestBlockInfo( // FIXME default value for hash ? + BestBlockInfo( // TODO ETCM-1090 provide the genesis hash as default get(Keys.BestBlockHash).map(v => ByteString(Hex.decode(v))).getOrElse(ByteString.empty), getBigInt(Keys.BestBlockNumber) ) diff --git a/src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala b/src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala index 3f2c4a84ea..643218fac2 100644 --- a/src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala +++ b/src/test/scala/io/iohk/ethereum/domain/BlockchainReaderSpec.scala @@ -1,19 +1,25 @@ package io.iohk.ethereum.domain +import org.bouncycastle.util.encoders.Hex import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -import io.iohk.ethereum.Fixtures +import io.iohk.ethereum.ObjectGenerators import io.iohk.ethereum.blockchain.sync.EphemBlockchainTestSetup +import io.iohk.ethereum.network.p2p.messages.BaseETH6XMessages.NewBlock +import io.iohk.ethereum.security.SecureRandomBuilder -class BlockchainReaderSpec extends AnyFlatSpec with Matchers { +class BlockchainReaderSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyChecks with SecureRandomBuilder { - "BlockchainReader" should "be able to get the best block after it was stored by BlockchainWriter" in new EphemBlockchainTestSetup { - val validBlock = Fixtures.Blocks.ValidBlock.block + val chainId: Option[Byte] = Hex.decode("3d").headOption - blockchainWriter.save(validBlock, Nil, ChainWeight.zero, true) + "BlockchainReader" should "be able to get the best block after it was stored by BlockchainWriter" in new EphemBlockchainTestSetup { + forAll(ObjectGenerators.newBlockGen(secureRandom, chainId)) { case NewBlock(block, weight) => + blockchainWriter.save(block, Nil, ChainWeight(0, weight), true) - blockchainReader.getBestBlock() shouldBe Some(validBlock) + blockchainReader.getBestBlock() shouldBe Some(block) + } } } From 1d57b4416c2c66dcc856876d49f3917cf4884c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Tue, 10 Aug 2021 10:47:24 +0200 Subject: [PATCH 08/11] fix warnings --- src/main/scala/io/iohk/ethereum/domain/Blockchain.scala | 1 - .../iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala | 1 - src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala | 1 - 3 files changed, 3 deletions(-) diff --git a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala index 3147ad1c7d..6733c94008 100644 --- a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala +++ b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala @@ -176,7 +176,6 @@ class BlockchainImpl( log.debug(s"Trying to remove block ${block.idTag}") val txList = block.body.transactionList - val bestBlockNumber = blockchainReader.getBestBlockNumber() val latestCheckpointNumber = getLatestCheckpointBlockNumber() val blockNumberMappingUpdates = diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala index b4f6bb6f0c..17754acc3e 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala @@ -1,6 +1,5 @@ package io.iohk.ethereum.blockchain.sync -import io.iohk.ethereum.Fixtures import io.iohk.ethereum.db.components.EphemDataSourceComponent import io.iohk.ethereum.db.components.Storages import io.iohk.ethereum.db.storage.pruning.ArchivePruning diff --git a/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala b/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala index fa9bd4c9df..763dcbf024 100644 --- a/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala +++ b/src/test/scala/io/iohk/ethereum/consensus/ConsensusSpec.scala @@ -22,7 +22,6 @@ import io.iohk.ethereum.consensus.validators.BlockHeaderError.HeaderParentNotFou import io.iohk.ethereum.consensus.validators._ import io.iohk.ethereum.db.storage.MptStorage import io.iohk.ethereum.domain._ -import io.iohk.ethereum.domain.appstate.BestBlockInfo import io.iohk.ethereum.ledger.BlockData import io.iohk.ethereum.ledger.BlockExecution import io.iohk.ethereum.ledger.BlockQueue.Leaf From beb908b4b1fc336f4e03633697293d5db6efe760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Tue, 10 Aug 2021 13:12:45 +0200 Subject: [PATCH 09/11] Add a fix for the database --- .../ethereum/domain/BlockchainReader.scala | 4 +++- .../iohk/ethereum/nodebuilder/StdNode.scala | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala b/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala index c97fb943bd..4eda16636c 100644 --- a/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala +++ b/src/main/scala/io/iohk/ethereum/domain/BlockchainReader.scala @@ -1,6 +1,7 @@ package io.iohk.ethereum.domain import akka.util.ByteString + import io.iohk.ethereum.db.storage.AppStateStorage import io.iohk.ethereum.db.storage.BlockBodiesStorage import io.iohk.ethereum.db.storage.BlockHeadersStorage @@ -12,7 +13,8 @@ import io.iohk.ethereum.domain.branch.Branch import io.iohk.ethereum.domain.branch.EmptyBranch import io.iohk.ethereum.mpt.MerklePatriciaTrie import io.iohk.ethereum.mpt.MptNode -import io.iohk.ethereum.utils.{Hex, Logger} +import io.iohk.ethereum.utils.Hex +import io.iohk.ethereum.utils.Logger class BlockchainReader( blockHeadersStorage: BlockHeadersStorage, diff --git a/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala b/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala index 784a9e2816..fd095b08bc 100644 --- a/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala +++ b/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala @@ -1,6 +1,7 @@ package io.iohk.ethereum.nodebuilder import akka.actor.typed.ActorSystem +import akka.util.ByteString import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global @@ -18,6 +19,7 @@ import io.iohk.ethereum.network.discovery.PeerDiscoveryManager import io.iohk.ethereum.nodebuilder.tooling.PeriodicConsistencyCheck import io.iohk.ethereum.nodebuilder.tooling.StorageConsistencyChecker import io.iohk.ethereum.utils.Config +import io.iohk.ethereum.utils.Hex /** A standard node is everything Ethereum prescribes except the mining algorithm, * which is plugged in dynamically. @@ -32,6 +34,8 @@ abstract class BaseNode extends Node { def start(): Unit = { startMetricsClient() + fixDatabase() + loadGenesisData() runDBConsistencyCheck() @@ -132,6 +136,24 @@ abstract class BaseNode extends Node { tryAndLogFailure(() => Metrics.get().close()) tryAndLogFailure(() => storagesInstance.dataSource.close()) } + + def fixDatabase(): Unit = { + // FIXME this is a temporary solution to avoid an incompatibility due to the introduction of the best block hash + // We can remove this fix when we release an incompatible version. + val bestBlockInfo = storagesInstance.storages.appStateStorage.getBestBlockInfo() + if (bestBlockInfo.hash == ByteString.empty && bestBlockInfo.number > 0) { + log.warn("Fixing best block hash into database for block {}", bestBlockInfo.number) + storagesInstance.storages.blockNumberMappingStorage.get(bestBlockInfo.number) match { + case Some(hash) => + log.warn("Putting {} as the best block hash", Hex.toHexString(hash.toArray)) + storagesInstance.storages.appStateStorage.putBestBlockInfo(bestBlockInfo.copy(hash = hash)).commit() + case None => + log.error("No block found for number {} when trying to fix database", bestBlockInfo.number) + } + + } + + } } class StdNode extends BaseNode with StdMiningBuilder From 2d1d1f382ada7aadb09e636b1b27fa9682fa1463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Richez?= Date: Wed, 11 Aug 2021 07:52:55 +0200 Subject: [PATCH 10/11] Rename BestBlockInfo to BlockInfo --- .../io/iohk/ethereum/db/storage/AppStateStorage.scala | 8 ++++---- src/main/scala/io/iohk/ethereum/domain/Blockchain.scala | 8 ++++---- .../scala/io/iohk/ethereum/domain/BlockchainWriter.scala | 6 +++--- .../appstate/{BestBlockInfo.scala => BlockInfo.scala} | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) rename src/main/scala/io/iohk/ethereum/domain/appstate/{BestBlockInfo.scala => BlockInfo.scala} (54%) diff --git a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala index 119ebb9da3..4339747f01 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala @@ -9,7 +9,7 @@ import scala.collection.immutable.ArraySeq import io.iohk.ethereum.db.dataSource.DataSource import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage.AppStateStorage._ -import io.iohk.ethereum.domain.appstate.BestBlockInfo +import io.iohk.ethereum.domain.appstate.BlockInfo import io.iohk.ethereum.utils.Hex /** This class is used to store app state variables @@ -31,13 +31,13 @@ class AppStateStorage(val dataSource: DataSource) extends TransactionalKeyValueS def getBestBlockNumber(): BigInt = getBigInt(Keys.BestBlockNumber) - def getBestBlockInfo(): BestBlockInfo = - BestBlockInfo( // TODO ETCM-1090 provide the genesis hash as default + def getBestBlockInfo(): BlockInfo = + BlockInfo( // TODO ETCM-1090 provide the genesis hash as default get(Keys.BestBlockHash).map(v => ByteString(Hex.decode(v))).getOrElse(ByteString.empty), getBigInt(Keys.BestBlockNumber) ) - def putBestBlockInfo(b: BestBlockInfo): DataSourceBatchUpdate = + def putBestBlockInfo(b: BlockInfo): DataSourceBatchUpdate = put(Keys.BestBlockNumber, b.number.toString) .and(put(Keys.BestBlockHash, Hex.toHexString(b.hash.toArray))) diff --git a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala index 6733c94008..6e9e003934 100644 --- a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala +++ b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala @@ -7,7 +7,7 @@ import scala.annotation.tailrec import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage._ import io.iohk.ethereum.domain -import io.iohk.ethereum.domain.appstate.BestBlockInfo +import io.iohk.ethereum.domain.appstate.BlockInfo import io.iohk.ethereum.jsonrpc.ProofService.StorageProof import io.iohk.ethereum.ledger.InMemoryWorldStateProxy import io.iohk.ethereum.ledger.InMemoryWorldStateProxyStorage @@ -144,7 +144,7 @@ class BlockchainImpl( } private def saveBestKnownBlock(bestBlockHash: ByteString, bestBlockNumber: BigInt): Unit = - appStateStorage.putBestBlockInfo(BestBlockInfo(bestBlockHash, bestBlockNumber)).commit() + appStateStorage.putBestBlockInfo(BlockInfo(bestBlockHash, bestBlockNumber)).commit() private def saveBestKnownBlockAndLatestCheckpointNumber( bestBlockHash: ByteString, @@ -152,7 +152,7 @@ class BlockchainImpl( latestCheckpointNumber: BigInt ): Unit = appStateStorage - .putBestBlockInfo(BestBlockInfo(bestBlockHash, number)) + .putBestBlockInfo(BlockInfo(bestBlockHash, number)) .and(appStateStorage.putLatestCheckpointBlockNumber(latestCheckpointNumber)) .commit() @@ -201,7 +201,7 @@ class BlockchainImpl( */ val bestBlockNumberUpdates = if (appStateStorage.getBestBlockNumber() > potentialNewBestBlockNumber) - appStateStorage.putBestBlockInfo(BestBlockInfo(potentialNewBestBlockHash, potentialNewBestBlockNumber)) + appStateStorage.putBestBlockInfo(BlockInfo(potentialNewBestBlockHash, potentialNewBestBlockNumber)) else appStateStorage.emptyBatchUpdate val latestCheckpointNumberUpdates = if (appStateStorage.getLatestCheckpointBlockNumber() > newLatestCheckpointNumber) diff --git a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala index 615992b05b..b5ec94da2a 100644 --- a/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala +++ b/src/main/scala/io/iohk/ethereum/domain/BlockchainWriter.scala @@ -11,7 +11,7 @@ import io.iohk.ethereum.db.storage.ChainWeightStorage import io.iohk.ethereum.db.storage.ReceiptStorage import io.iohk.ethereum.db.storage.TransactionMappingStorage import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation -import io.iohk.ethereum.domain.appstate.BestBlockInfo +import io.iohk.ethereum.domain.appstate.BlockInfo import io.iohk.ethereum.utils.Logger class BlockchainWriter( @@ -32,14 +32,14 @@ class BlockchainWriter( block.header.number ) appStateStorage - .putBestBlockInfo(BestBlockInfo(block.header.hash, block.header.number)) + .putBestBlockInfo(BlockInfo(block.header.hash, block.header.number)) .and(appStateStorage.putLatestCheckpointBlockNumber(block.header.number)) } else if (saveAsBestBlock) { log.debug( "New best known block number - {}", block.header.number ) - appStateStorage.putBestBlockInfo(BestBlockInfo(block.header.hash, block.header.number)) + appStateStorage.putBestBlockInfo(BlockInfo(block.header.hash, block.header.number)) } else { appStateStorage.emptyBatchUpdate } diff --git a/src/main/scala/io/iohk/ethereum/domain/appstate/BestBlockInfo.scala b/src/main/scala/io/iohk/ethereum/domain/appstate/BlockInfo.scala similarity index 54% rename from src/main/scala/io/iohk/ethereum/domain/appstate/BestBlockInfo.scala rename to src/main/scala/io/iohk/ethereum/domain/appstate/BlockInfo.scala index 45a6cbfdd0..76359ae038 100644 --- a/src/main/scala/io/iohk/ethereum/domain/appstate/BestBlockInfo.scala +++ b/src/main/scala/io/iohk/ethereum/domain/appstate/BlockInfo.scala @@ -2,4 +2,4 @@ package io.iohk.ethereum.domain.appstate import akka.util.ByteString -case class BestBlockInfo(hash: ByteString, number: BigInt) +case class BlockInfo(hash: ByteString, number: BigInt) From 6b3202feb06afd98cf37f222cb09818dd65bfbd9 Mon Sep 17 00:00:00 2001 From: Leonor Boga Date: Tue, 10 Aug 2021 14:22:15 +0200 Subject: [PATCH 11/11] [cherry pick from ETCM-1045] Fix Blockchain test about rolling back blocks --- .../io/iohk/ethereum/ObjectGenerators.scala | 2 -- .../iohk/ethereum/domain/BlockchainSpec.scala | 34 +++---------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/src/test/scala/io/iohk/ethereum/ObjectGenerators.scala b/src/test/scala/io/iohk/ethereum/ObjectGenerators.scala index 3269e557a1..72ccfadc09 100644 --- a/src/test/scala/io/iohk/ethereum/ObjectGenerators.scala +++ b/src/test/scala/io/iohk/ethereum/ObjectGenerators.scala @@ -33,8 +33,6 @@ trait ObjectGenerators { def intGen(min: Int, max: Int): Gen[Int] = Gen.choose(min, max) - def posIntGen(min: Int, max: Int): Gen[Int] = Gen.choose(min, max).suchThat(_ > 0) - def intGen: Gen[Int] = Gen.choose(Int.MinValue, Int.MaxValue) def longGen: Gen[Long] = Gen.choose(Long.MinValue, Long.MaxValue) diff --git a/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala b/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala index b0feb8a99d..b3d6e42f08 100644 --- a/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala +++ b/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala @@ -214,7 +214,7 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh } it should "return correct best block number after saving and rolling back blocks" in new TestSetup { - forAll(posIntGen(min = 1, max = maxNumberBlocksToImport)) { numberBlocksToImport => + forAll(intGen(min = 1, max = maxNumberBlocksToImport)) { numberBlocksToImport => val testSetup = newSetup() import testSetup._ @@ -237,8 +237,9 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh blockchainReaderWithStubPersisting.getBestBlockNumber() shouldBe blocksToImport.last.number // Rollback blocks - val numberBlocksToRollback = intGen(0, numberBlocksToImport).sample.get - val (_, blocksToRollback) = blocksToImport.splitAt(numberBlocksToRollback) + val numberBlocksToKeep = intGen(0, numberBlocksToImport).sample.get + + val (_, blocksToRollback) = blocksToImport.splitAt(numberBlocksToKeep) // Randomly select the block rollback to persist (empty means no persistence) val blockRollbackToPersist = @@ -254,38 +255,13 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh blockchainWithStubPersisting.removeBlock(block.hash) } - val expectedPersistedBestBlock = calculatePersistedBestBlock( - blockImportToPersist.map(_.number), - blockRollbackToPersist.map(_.number), - blocksToRollback.map(_.number) - ) - blockchainReaderWithStubPersisting.getBestBlockNumber() shouldBe expectedPersistedBestBlock + blockchainReaderWithStubPersisting.getBestBlockNumber() shouldBe numberBlocksToKeep } } trait TestSetup extends MockFactory { val maxNumberBlocksToImport: Int = 30 - def calculatePersistedBestBlock( - blockImportPersisted: Option[BigInt], - blockRollbackPersisted: Option[BigInt], - blocksRolledback: Seq[BigInt] - ): BigInt = - (blocksRolledback, blockImportPersisted) match { - case (Nil, Some(bi)) => - // No blocks rolledback, last persist was the persist during import - bi - case (nonEmptyRolledbackBlocks, Some(bi)) => - // Last forced persist during apply/rollback - val maxForcedPersist = blockRollbackPersisted.fold(bi)(br => (br - 1).max(bi)) - - // The above number would have been decreased by any rollbacked blocks - (nonEmptyRolledbackBlocks.head - 1).min(maxForcedPersist) - case (_, None) => - // If persisted rollback, then it was decreased by the future rollbacks, if not no persistance was ever done - blockRollbackPersisted.fold(0: BigInt)(_ => blocksRolledback.head - 1) - } - trait StubPersistingBlockchainSetup { def stubStateStorage: StateStorage def blockchainStoragesWithStubPersisting: BlockchainStorages