[ETCM-943] partially split reading part of Blockchain

src/it/scala/io/iohk/ethereum/ledger/BlockImporterItSpec.scala

@@ -62,7 +62,7 @@ class BlockImporterItSpec
   val emptyWorld = InMemoryWorldStateProxy(
     storagesInstance.storages.evmCodeStorage,
     blockchain.getBackingMptStorage(-1),
-    (number: BigInt) => blockchain.getBlockHeaderByNumber(number).map(_.hash),
+    (number: BigInt) => blockchainReader.getBlockHeaderByNumber(number).map(_.hash),
     blockchainConfig.accountStartNonce,
     ByteString(MerklePatriciaTrie.EmptyRootHash),
     noEmptyAccounts = false,
@@ -85,6 +85,7 @@ class BlockImporterItSpec
     override private[ledger] lazy val blockExecution =
       new BlockExecution(
         blockchain,
+        blockchainReader,
         storagesInstance.storages.evmCodeStorage,
         blockchainConfig,
         consensus.blockPreparator,

src/it/scala/io/iohk/ethereum/sync/RegularSyncItSpec.scala

@@ -127,7 +127,7 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
       _ <- peer2.importBlocksUntil(30)(IdentityUpdate)
       _ <- peer1.startRegularSync()
       _ <- peer2.startRegularSync()
-      _ <- peer2.addCheckpointedBlock(peer2.bl.getBlockByNumber(25).get)
+      _ <- peer2.addCheckpointedBlock(peer2.blockchainReader.getBlockByNumber(25).get)
       _ <- peer2.waitForRegularSyncLoadLastBlock(length)
       _ <- peer1.connectToPeers(Set(peer2.node))
       _ <- peer1.waitForRegularSyncLoadLastBlock(length)
@@ -188,7 +188,10 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
           peer2.bl.getBestBlock().get.hash
         )
       )
-      (peer1.bl.getBlockByNumber(blockNumer + 1), peer2.bl.getBlockByNumber(blockNumer + 1)) match {
+      (
+        peer1.blockchainReader.getBlockByNumber(blockNumer + 1),
+        peer2.blockchainReader.getBlockByNumber(blockNumer + 1)
+      ) match {
         case (Some(blockP1), Some(blockP2)) =>
           assert(peer1.bl.getChainWeightByHash(blockP1.hash) == peer2.bl.getChainWeightByHash(blockP2.hash))
         case (_, _) => fail("invalid difficulty validation")

src/it/scala/io/iohk/ethereum/sync/util/CommonFakePeer.scala

@@ -14,7 +14,7 @@ import io.iohk.ethereum.db.components.{RocksDbDataSourceComponent, Storages}
 import io.iohk.ethereum.db.dataSource.{RocksDbConfig, RocksDbDataSource}
 import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
 import io.iohk.ethereum.db.storage.{AppStateStorage, Namespaces}
-import io.iohk.ethereum.domain.{Block, Blockchain, BlockchainImpl, ChainWeight, UInt256}
+import io.iohk.ethereum.domain.{Block, Blockchain, BlockchainImpl, BlockchainReader, ChainWeight, UInt256}
 import io.iohk.ethereum.security.SecureRandomBuilder
 import io.iohk.ethereum.ledger.InMemoryWorldStateProxy
 import io.iohk.ethereum.mpt.MerklePatriciaTrie
@@ -115,7 +115,8 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
     )
   )
 
-  val bl = BlockchainImpl(storagesInstance.storages)
+  val blockchainReader = BlockchainReader(storagesInstance.storages)
+  val bl = BlockchainImpl(storagesInstance.storages, blockchainReader)
   val evmCodeStorage = storagesInstance.storages.evmCodeStorage
 
   val genesis = Block(
@@ -168,7 +169,8 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
       override val forkResolverOpt: Option[ForkResolver] = None
       override val nodeStatusHolder: AtomicReference[NodeStatus] = nh
       override val peerConfiguration: PeerConfiguration = peerConf
-      override val blockchain: Blockchain = bl
+      override val blockchain: Blockchain = CommonFakePeer.this.bl
+      override val blockchainReader: BlockchainReader = CommonFakePeer.this.blockchainReader
       override val appStateStorage: AppStateStorage = storagesInstance.storages.appStateStorage
       override val capabilities: List[Capability] = blockchainConfig.capabilities
     }
@@ -205,7 +207,8 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
 
   val blockchainHost: ActorRef =
     system.actorOf(
-      BlockchainHostActor.props(bl, storagesInstance.storages.evmCodeStorage, peerConf, peerEventBus, etcPeerManager),
+      BlockchainHostActor
+        .props(blockchainReader, storagesInstance.storages.evmCodeStorage, peerConf, peerEventBus, etcPeerManager),
       "blockchain-host"
     )
 
@@ -244,7 +247,7 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
     InMemoryWorldStateProxy(
       storagesInstance.storages.evmCodeStorage,
       bl.getBackingMptStorage(block.number),
-      (number: BigInt) => bl.getBlockHeaderByNumber(number).map(_.hash),
+      (number: BigInt) => blockchainReader.getBlockHeaderByNumber(number).map(_.hash),
       blockchainConfig.accountStartNonce,
       block.header.stateRoot,
       noEmptyAccounts = EvmConfig.forBlock(block.number, blockchainConfig).noEmptyAccounts,

src/it/scala/io/iohk/ethereum/sync/util/FastSyncItSpecUtils.scala

@@ -29,6 +29,7 @@ object FastSyncItSpecUtils {
         storagesInstance.storages.fastSyncStateStorage,
         storagesInstance.storages.appStateStorage,
         bl,
+        blockchainReader,
         storagesInstance.storages.evmCodeStorage,
         storagesInstance.storages.nodeStorage,
         validators,

src/it/scala/io/iohk/ethereum/sync/util/RegularSyncItSpecUtils.scala

@@ -65,6 +65,7 @@ object RegularSyncItSpecUtils {
           vm,
           storagesInstance.storages.evmCodeStorage,
           bl,
+          blockchainReader,
           blockchainConfig,
           fullConfig,
           ValidatorsExecutorAlwaysSucceed,
@@ -83,14 +84,15 @@ object RegularSyncItSpecUtils {
     lazy val ledger: Ledger =
       new LedgerImpl(
         bl,
+        blockchainReader,
         storagesInstance.storages.evmCodeStorage,
         blockchainConfig,
         syncConfig,
         buildEthashConsensus(),
         Scheduler.global
       )
 
-    lazy val ommersPool: ActorRef = system.actorOf(OmmersPool.props(bl, 1), "ommers-pool")
+    lazy val ommersPool: ActorRef = system.actorOf(OmmersPool.props(blockchainReader, 1), "ommers-pool")
 
     lazy val pendingTransactionsManager: ActorRef = system.actorOf(
       PendingTransactionsManager.props(TxPoolConfig(config), peerManager, etcPeerManager, peerEventBus),
@@ -156,7 +158,9 @@ object RegularSyncItSpecUtils {
     )(updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy): Task[Unit] = {
       Task(blockNumber match {
         case Some(bNumber) =>
-          bl.getBlockByNumber(bNumber).getOrElse(throw new RuntimeException(s"block by number: $bNumber doesn't exist"))
+          blockchainReader
+            .getBlockByNumber(bNumber)
+            .getOrElse(throw new RuntimeException(s"block by number: $bNumber doesn't exist"))
         case None => bl.getBestBlock().get
       }).flatMap { block =>
         Task {
@@ -215,7 +219,7 @@ object RegularSyncItSpecUtils {
       InMemoryWorldStateProxy(
         storagesInstance.storages.evmCodeStorage,
         bl.getBackingMptStorage(block.number),
-        (number: BigInt) => bl.getBlockHeaderByNumber(number).map(_.hash),
+        (number: BigInt) => blockchainReader.getBlockHeaderByNumber(number).map(_.hash),
         UInt256.Zero,
         ByteString(MerklePatriciaTrie.EmptyRootHash),
         noEmptyAccounts = false,

src/it/scala/io/iohk/ethereum/txExecTest/ContractTest.scala

@@ -1,6 +1,6 @@
 package io.iohk.ethereum.txExecTest
 
-import io.iohk.ethereum.domain.{BlockchainImpl, Receipt}
+import io.iohk.ethereum.domain.{BlockchainImpl, BlockchainReader, Receipt}
 import io.iohk.ethereum.ledger.{BlockExecution, BlockQueue, BlockValidation}
 import io.iohk.ethereum.txExecTest.util.FixtureProvider
 import io.iohk.ethereum.utils.Config
@@ -16,19 +16,18 @@ class ContractTest extends AnyFlatSpec with Matchers {
     val fixtures: FixtureProvider.Fixture = FixtureProvider.loadFixtures("/txExecTest/purchaseContract")
     lazy val testBlockchainStorages = FixtureProvider.prepareStorages(2, fixtures)
 
-    override lazy val blockchain = BlockchainImpl(testBlockchainStorages)
-
     //block only with ether transfers
-    val blockValidation = new BlockValidation(consensus, blockchain, BlockQueue(blockchain, syncConfig))
-    val blockExecution = new BlockExecution(
-      blockchain,
-      testBlockchainStorages.evmCodeStorage,
-      blockchainConfig,
-      consensus.blockPreparator,
-      blockValidation
-    )
-
-    // transfer ether
+    val blockValidation =
+      new BlockValidation(consensus, blockchainReader, BlockQueue(blockchain, syncConfig))
+    val blockExecution =
+      new BlockExecution(
+        blockchain,
+        blockchainReader,
+        testBlockchainStorages.evmCodeStorage,
+        blockchainConfig,
+        consensus.blockPreparator,
+        blockValidation
+      )
     blockExecution.executeAndValidateBlock(fixtures.blockByNumber(1)) shouldBe noErrors
 
     // deploy contract

src/it/scala/io/iohk/ethereum/txExecTest/ECIP1017Test.scala

@@ -1,6 +1,6 @@
 package io.iohk.ethereum.txExecTest
 
-import io.iohk.ethereum.domain.{Address, Receipt, UInt256}
+import io.iohk.ethereum.domain.{Address, BlockchainImpl, BlockchainReader, Receipt, UInt256}
 import io.iohk.ethereum.ledger.{BlockExecution, BlockQueue, BlockValidation}
 import io.iohk.ethereum.txExecTest.util.FixtureProvider
 import io.iohk.ethereum.utils.{BlockchainConfig, ForkBlockNumbers, MonetaryPolicyConfig}
@@ -68,14 +68,20 @@ class ECIP1017Test extends AnyFlatSpec with Matchers {
     protected val testBlockchainStorages = FixtureProvider.prepareStorages(endBlock, fixtures)
 
     (startBlock to endBlock) foreach { blockToExecute =>
-      val blockValidation = new BlockValidation(consensus, blockchain, BlockQueue(blockchain, syncConfig))
-      val blockExecution = new BlockExecution(
-        blockchain,
-        testBlockchainStorages.evmCodeStorage,
-        blockchainConfig,
-        consensus.blockPreparator,
-        blockValidation
-      )
+      val storages = FixtureProvider.prepareStorages(blockToExecute - 1, fixtures)
+      val blockchainReader = BlockchainReader(storages)
+      val blockchain = BlockchainImpl(storages, blockchainReader)
+      val blockValidation =
+        new BlockValidation(consensus, blockchainReader, BlockQueue(blockchain, syncConfig))
+      val blockExecution =
+        new BlockExecution(
+          blockchain,
+          blockchainReader,
+          testBlockchainStorages.evmCodeStorage,
+          blockchainConfig,
+          consensus.blockPreparator,
+          blockValidation
+        )
       blockExecution.executeAndValidateBlock(fixtures.blockByNumber(blockToExecute)) shouldBe noErrors
     }
   }

src/it/scala/io/iohk/ethereum/txExecTest/ForksTest.scala

@@ -1,6 +1,7 @@
 package io.iohk.ethereum.txExecTest
 
-import io.iohk.ethereum.domain.{Address, Receipt, UInt256}
+import java.util.concurrent.Executors
+import io.iohk.ethereum.domain.{Address, BlockchainImpl, BlockchainReader, Receipt, UInt256}
 import io.iohk.ethereum.ledger.{BlockExecution, BlockQueue, BlockValidation}
 import io.iohk.ethereum.txExecTest.util.FixtureProvider
 import io.iohk.ethereum.utils.{BlockchainConfig, ForkBlockNumbers, MonetaryPolicyConfig}
@@ -60,14 +61,20 @@ class ForksTest extends AnyFlatSpec with Matchers {
     protected val testBlockchainStorages = FixtureProvider.prepareStorages(endBlock, fixtures)
 
     (startBlock to endBlock) foreach { blockToExecute =>
-      val blockValidation = new BlockValidation(consensus, blockchain, BlockQueue(blockchain, syncConfig))
-      val blockExecution = new BlockExecution(
-        blockchain,
-        testBlockchainStorages.evmCodeStorage,
-        blockchainConfig,
-        consensus.blockPreparator,
-        blockValidation
-      )
+      val storages = FixtureProvider.prepareStorages(blockToExecute - 1, fixtures)
+      val blockchainReader = BlockchainReader(storages)
+      val blockchain = BlockchainImpl(storages, blockchainReader)
+      val blockValidation =
+        new BlockValidation(consensus, blockchainReader, BlockQueue(blockchain, syncConfig))
+      val blockExecution =
+        new BlockExecution(
+          blockchain,
+          blockchainReader,
+          testBlockchainStorages.evmCodeStorage,
+          blockchainConfig,
+          consensus.blockPreparator,
+          blockValidation
+        )
       blockExecution.executeAndValidateBlock(fixtures.blockByNumber(blockToExecute)) shouldBe noErrors
     }
   }

src/it/scala/io/iohk/ethereum/txExecTest/ScenarioSetup.scala

@@ -1,11 +1,13 @@
 package io.iohk.ethereum.txExecTest
 
 import io.iohk.ethereum.blockchain.sync.EphemBlockchainTestSetup
-import io.iohk.ethereum.domain.{BlockchainImpl, BlockchainStorages}
+import io.iohk.ethereum.domain.{BlockchainImpl, BlockchainReader, BlockchainStorages}
 import io.iohk.ethereum.ledger.Ledger.VMImpl
 
 trait ScenarioSetup extends EphemBlockchainTestSetup {
   protected val testBlockchainStorages: BlockchainStorages
-  override lazy val blockchain: BlockchainImpl = BlockchainImpl(testBlockchainStorages)
+
+  override lazy val blockchainReader: BlockchainReader = BlockchainReader(testBlockchainStorages)
+  override lazy val blockchain: BlockchainImpl = BlockchainImpl(testBlockchainStorages, blockchainReader)
   override lazy val vm: VMImpl = new VMImpl
 }

src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

@@ -8,34 +8,37 @@ import com.typesafe.config.ConfigFactory
 import io.iohk.ethereum.blockchain.sync.CacheBasedBlacklist
 import io.iohk.ethereum.db.components.Storages.PruningModeComponent
 import io.iohk.ethereum.db.components.{RocksDbDataSourceComponent, Storages}
-import io.iohk.ethereum.db.dataSource.{DataSourceBatchUpdate, RocksDbDataSource}
+import io.iohk.ethereum.db.dataSource.{DataSourceBatchUpdate}
 import io.iohk.ethereum.db.storage.NodeStorage.{NodeEncoded, NodeHash}
-import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation
 import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
 import io.iohk.ethereum.db.storage.{AppStateStorage, MptStorage, StateStorage}
 import io.iohk.ethereum.domain.BlockHeader.HeaderExtraFields.HefEmpty
 import io.iohk.ethereum.domain.{Blockchain, UInt256, _}
-import io.iohk.ethereum.jsonrpc.ProofService.{EmptyStorageValueProof, StorageProof, StorageProofKey, StorageValueProof}
+import io.iohk.ethereum.jsonrpc.ProofService.{EmptyStorageValueProof, StorageProof, StorageProofKey}
 import io.iohk.ethereum.ledger.{InMemoryWorldStateProxy, InMemoryWorldStateProxyStorage}
 import io.iohk.ethereum.mpt.MptNode
 import io.iohk.ethereum.network.EtcPeerManagerActor.PeerInfo
 import io.iohk.ethereum.network.PeerManagerActor.PeerConfiguration
 import io.iohk.ethereum.network.PeerStatisticsActor
 import io.iohk.ethereum.network.discovery.DiscoveryConfig
 import io.iohk.ethereum.network.handshaker.{EtcHandshaker, EtcHandshakerConfiguration, Handshaker}
-import io.iohk.ethereum.network.p2p.EthereumMessageDecoder
 import io.iohk.ethereum.network.p2p.messages.Capability
 import io.iohk.ethereum.network.rlpx.RLPxConnectionHandler.RLPxConfiguration
 import io.iohk.ethereum.network.{ForkResolver, PeerEventBusActor, PeerManagerActor}
 import io.iohk.ethereum.nodebuilder.{AuthHandshakerBuilder, NodeKeyBuilder}
 import io.iohk.ethereum.security.SecureRandomBuilder
 import io.iohk.ethereum.utils.{Config, NodeStatus, ServerStatus}
-import monix.reactive.Observable
 import org.bouncycastle.util.encoders.Hex
+import org.scalamock.scalatest.MockFactory
 
 import scala.concurrent.duration._
 
-object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder with AuthHandshakerBuilder {
+object DumpChainApp
+    extends App
+    with NodeKeyBuilder
+    with SecureRandomBuilder
+    with AuthHandshakerBuilder
+    with MockFactory {
   val conf = ConfigFactory.load("txExecTest/chainDump.conf")
   val node = conf.getString("node")
   val genesisHash = ByteString(Hex.decode(conf.getString("genesisHash")))
@@ -78,6 +81,8 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit
   val storagesInstance = new RocksDbDataSourceComponent with PruningConfig with Storages.DefaultStorages
 
   val blockchain: Blockchain = new BlockchainMock(genesisHash)
+  val blockchainReader = mock[BlockchainReader]
+  (blockchainReader.getHashByBlockNumber _).expects(*).returning(Some(genesisHash))
 
   val nodeStatus =
     NodeStatus(key = nodeKey, serverStatus = ServerStatus.NotListening, discoveryStatus = ServerStatus.NotListening)
@@ -92,6 +97,7 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit
       override val nodeStatusHolder: AtomicReference[NodeStatus] = DumpChainApp.nodeStatusHolder
       override val peerConfiguration: PeerConfiguration = peerConfig
       override val blockchain: Blockchain = DumpChainApp.blockchain
+      override val blockchainReader: BlockchainReader = DumpChainApp.blockchainReader
       override val appStateStorage: AppStateStorage = storagesInstance.storages.appStateStorage
       override val capabilities: List[Capability] = blockchainConfig.capabilities
     }
@@ -156,14 +162,6 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
       ethCompatibleStorage: Boolean
   ): StorageProof = EmptyStorageValueProof(StorageProofKey(position))
 
-  override protected def getHashByBlockNumber(number: BigInt): Option[ByteString] = Some(genesisHash)
-
-  override def getBlockHeaderByHash(hash: ByteString): Option[BlockHeader] = Some(new FakeHeader())
-
-  override def getBlockBodyByHash(hash: ByteString): Option[BlockBody] = ???
-
-  override def getMptNodeByHash(hash: ByteString): Option[MptNode] = ???
-
   override def storeBlockHeader(blockHeader: BlockHeader): DataSourceBatchUpdate = ???
 
   override def storeBlockBody(blockHash: ByteString, blockBody: BlockBody): DataSourceBatchUpdate = ???
@@ -180,8 +178,6 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
 
   override def getChainWeightByHash(blockhash: ByteString): Option[ChainWeight] = ???
 
-  override def getReceiptsByHash(blockhash: ByteString): Option[Seq[Receipt]] = ???
-
   def getAccount(address: Address, blockNumber: BigInt): Option[Account] = ???
 
   override def getAccountStorageAt(rootHash: ByteString, position: BigInt, ethCompatibleStorage: Boolean): ByteString =
@@ -202,6 +198,12 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
 
   override def getLatestCheckpointBlockNumber(): BigInt = ???
 
+  override def isInChain(hash: NodeHash): Boolean = ???
+
+  override def genesisHeader: BlockHeader = ???
+
+  override def genesisBlock: Block = ???
+
   override def getBackingMptStorage(blockNumber: BigInt): MptStorage = ???
 
   override def getReadOnlyMptStorage(): MptStorage = ???

src/main/scala/io/iohk/ethereum/blockchain/data/GenesisDataLoader.scala

@@ -23,8 +23,12 @@ import org.bouncycastle.util.encoders.Hex
 import scala.io.Source
 import scala.util.{Failure, Success, Try}
 
-class GenesisDataLoader(blockchain: Blockchain, stateStorage: StateStorage, blockchainConfig: BlockchainConfig)
-    extends Logger {
+class GenesisDataLoader(
+    blockchain: Blockchain,
+    blockchainReader: BlockchainReader,
+    stateStorage: StateStorage,
+    blockchainConfig: BlockchainConfig
+) extends Logger {
 
   private val bloomLength = 512
   private val hashLength = 64
@@ -99,7 +103,7 @@ class GenesisDataLoader(blockchain: Blockchain, stateStorage: StateStorage, bloc
 
     log.debug(s"Prepared genesis header: $header")
 
-    blockchain.getBlockHeaderByNumber(0) match {
+    blockchainReader.getBlockHeaderByNumber(0) match {
       case Some(existingGenesisHeader) if existingGenesisHeader.hash == header.hash =>
         log.debug("Genesis data already in the database")
         Success(())