diff --git a/src/benchmark/scala/io/iohk/ethereum/mpt/MerklePatriciaTreeSpeedSpec.scala b/src/benchmark/scala/io/iohk/ethereum/mpt/MerklePatriciaTreeSpeedSpec.scala index aac1c530b9..f1e996bf6b 100644 --- a/src/benchmark/scala/io/iohk/ethereum/mpt/MerklePatriciaTreeSpeedSpec.scala +++ b/src/benchmark/scala/io/iohk/ethereum/mpt/MerklePatriciaTreeSpeedSpec.scala @@ -1,12 +1,12 @@ package io.iohk.ethereum.mpt -import io.iohk.ethereum.{ObjectGenerators, crypto} import io.iohk.ethereum.db.dataSource.EphemDataSource import io.iohk.ethereum.db.storage.{ArchiveNodeStorage, MptStorage, NodeStorage, SerializingMptStorage} import io.iohk.ethereum.mpt.MerklePatriciaTrie.defaultByteArraySerializable import io.iohk.ethereum.utils.Logger -import org.scalatest.FunSuite +import io.iohk.ethereum.{ObjectGenerators, crypto} import org.bouncycastle.util.encoders.Hex +import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks class MerklePatriciaTreeSpeedSpec diff --git a/src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala b/src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala index 31d40b6925..267bbf87f5 100644 --- a/src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala +++ b/src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala @@ -6,7 +6,7 @@ import io.iohk.ethereum.consensus.ethash.EthashConsensus import io.iohk.ethereum.consensus.ethash.validators.ValidatorsExecutor import io.iohk.ethereum.consensus.{ConsensusConfig, FullConsensusConfig, TestConsensus, ethash} import io.iohk.ethereum.db.components.Storages.PruningModeComponent -import io.iohk.ethereum.db.components.{SharedEphemDataSources, Storages} +import io.iohk.ethereum.db.components.{EphemDataSourceComponent, Storages} import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode} import io.iohk.ethereum.domain.Block.BlockDec import io.iohk.ethereum.domain._ @@ -36,7 +36,7 @@ object ScenarioSetup { def getBlockchain: BlockchainImpl = { - val storagesInstance = new SharedEphemDataSources with Pruning with Storages.DefaultStorages + val storagesInstance = new EphemDataSourceComponent with Pruning with Storages.DefaultStorages BlockchainImpl(storagesInstance.storages) } } @@ -75,9 +75,11 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) { Block(scenario.genesisBlockHeader.toBlockHeader, BlockBody(Nil, Nil)) } - blockchain.save(genesisBlock) - blockchain.save(genesisBlock.header.hash, Nil) - blockchain.save(genesisBlock.header.hash, genesisBlock.header.difficulty) + blockchain.storeBlock(genesisBlock) + .and(blockchain.storeReceipts(genesisBlock.header.hash, Nil)) + .and(blockchain.storeTotalDifficulty(genesisBlock.header.hash, genesisBlock.header.difficulty)) + .commit() + genesisBlock } diff --git a/src/it/scala/io/iohk/ethereum/db/DataSourceIntegrationTestBehavior.scala b/src/it/scala/io/iohk/ethereum/db/DataSourceIntegrationTestBehavior.scala index c7501be254..ff454aecc6 100644 --- a/src/it/scala/io/iohk/ethereum/db/DataSourceIntegrationTestBehavior.scala +++ b/src/it/scala/io/iohk/ethereum/db/DataSourceIntegrationTestBehavior.scala @@ -4,7 +4,8 @@ import java.io.File import java.nio.file.Files import akka.util.ByteString import io.iohk.ethereum.ObjectGenerators -import io.iohk.ethereum.db.dataSource.DataSource +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdate} +import io.iohk.ethereum.db.dataSource.DataSource.{Key, Namespace, Value} import org.scalatest.FlatSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks @@ -30,12 +31,19 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob } } + def prepareUpdate( + namespace: Namespace = OtherNamespace, + toRemove: Seq[Key] = Nil, + toUpsert: Seq[(Key, Value)] = Nil + ): Seq[DataSourceUpdate] = + Seq(DataSourceUpdate(namespace, toRemove, toUpsert)) + def updateInSeparateCalls( - dataSource: DataSource, - toUpsert: Seq[(ByteString, ByteString)] - ): DataSource = { - toUpsert.foldLeft(dataSource) { case (recDB, keyValuePair) => - recDB.update(OtherNamespace, Seq(), Seq(keyValuePair)) + dataSource: DataSource, + toUpsert: Seq[(ByteString, ByteString)] + ): Unit = { + toUpsert.foreach { keyValuePair => + dataSource.update(prepareUpdate(toUpsert = Seq(keyValuePair))) } } @@ -45,8 +53,9 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] => withDir { path => val keyList = unFilteredKeyList.take(KeyNumberLimit) - val db = updateInSeparateCalls( - dataSource = createDataSource(path), + val db = createDataSource(path) + updateInSeparateCalls( + dataSource = db, toUpsert = keyList.zip(keyList) ) keyList.foreach { key => @@ -62,11 +71,8 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] => withDir { path => val keyList = unFilteredKeyList.take(KeyNumberLimit) - val db = createDataSource(path).update( - OtherNamespace, - Seq(), - keyList.zip(keyList) - ) + val db = createDataSource(path) + db.update(prepareUpdate(toUpsert = keyList.zip(keyList))) keyList.foreach { key => assert(db.get(OtherNamespace, key).contains(key)) @@ -81,21 +87,18 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] => withDir { path => val keyList = unFilteredKeyList.take(KeyNumberLimit) - val db = createDataSource(path).update( - OtherNamespace, - Seq(), - keyList.zip(keyList) - ) + val db = createDataSource(path) + db.update(prepareUpdate(toUpsert = keyList.zip(keyList))) val keyListWithExtraByte = keyList.map(1.toByte +: _) - val dbAfterUpdate = - updateInSeparateCalls(db, keyList.zip(keyListWithExtraByte)) + updateInSeparateCalls(db, keyList.zip(keyListWithExtraByte)) - keyList.zip(keyListWithExtraByte).foreach { case (key, value) => - assert(dbAfterUpdate.get(OtherNamespace, key).contains(value)) + keyList.zip(keyListWithExtraByte).foreach { + case (key, value) => + assert(db.get(OtherNamespace, key).contains(value)) } - dbAfterUpdate.destroy() + db.destroy() } } } @@ -104,24 +107,18 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] => withDir { path => val keyList = unFilteredKeyList.take(KeyNumberLimit) - val db = createDataSource(path).update( - OtherNamespace, - Seq(), - keyList.zip(keyList) - ) + val db = createDataSource(path) + db.update(prepareUpdate(toUpsert = keyList.zip(keyList))) val keyListWithExtraByte = keyList.map(1.toByte +: _) - val dbAfterUpdate = db.update( - OtherNamespace, - Seq(), - keyList.zip(keyListWithExtraByte) - ) + db.update(prepareUpdate(toUpsert = keyList.zip(keyListWithExtraByte))) - keyList.zip(keyListWithExtraByte).foreach { case (key, value) => - assert(dbAfterUpdate.get(OtherNamespace, key).contains(value)) + keyList.zip(keyListWithExtraByte).foreach { + case (key, value) => + assert(db.get(OtherNamespace, key).contains(value)) } - dbAfterUpdate.destroy() + db.destroy() } } } @@ -131,12 +128,8 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob withDir { path => val keyList = unFilteredKeyList.take(KeyNumberLimit) val db = createDataSource(path) - .update( - namespace = OtherNamespace, - toRemove = Seq(), - toUpsert = keyList.zip(keyList) - ) - .clear + db.update(prepareUpdate(toUpsert = keyList.zip(keyList))) + db.clear() keyList.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) @@ -151,11 +144,8 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] => withDir { path => val keyList = unFilteredKeyList.take(KeyNumberLimit) - val db = createDataSource(path).update( - namespace = OtherNamespace, - toRemove = Seq(), - toUpsert = keyList.zip(keyList) - ) + val db = createDataSource(path) + db.update(prepareUpdate(toUpsert = keyList.zip(keyList))) db.close() val dbAfterClose = createDataSource(path) @@ -172,14 +162,11 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob withDir { path => forAll(seqByteStringOfNItemsGen(KeySizeWithoutPrefix)) { unFilteredKeyList: Seq[ByteString] => val keyList = unFilteredKeyList.take(KeyNumberLimit) - val db = createDataSource(path).update( - namespace = OtherNamespace, - toRemove = Seq(), - toUpsert = keyList.zip(keyList) - ) + val db = createDataSource(path) + db.update(prepareUpdate(toUpsert = keyList.zip(keyList))) db.destroy() - assert(!new File(path).exists()) + assert(!new File("/tmp/iodbDestroy").exists()) val dbAfterDestroy = createDataSource(path) keyList.foreach { key => @@ -199,15 +186,15 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob val db = createDataSource(path) val valList1 = keyList.map(1.toByte +: _) - db.update(OtherNamespace, Seq(), keyList.zip(valList1)) + db.update(prepareUpdate(namespace = OtherNamespace, toUpsert = keyList.zip(valList1))) val valList2 = keyList.map(2.toByte +: _) - db.update(OtherNamespace2, Seq(), keyList.zip(valList2)) + db.update(prepareUpdate(namespace = OtherNamespace2, toUpsert = keyList.zip(valList2))) - keyList.zip(valList1).foreach { case (key, value) => - assert(db.get(OtherNamespace, key).contains(value)) + keyList.zip(valList1).foreach { + case (key, value) => + assert(db.get(OtherNamespace, key).contains(value)) } - keyList.zip(valList2).foreach { case (key, value) => assert(db.get(OtherNamespace2, key).contains(value)) } @@ -225,23 +212,24 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob val db = createDataSource(path) val valList1 = keyList.map(1.toByte +: _) - db.update(OtherNamespace, Seq(), keyList.zip(valList1)) + db.update(prepareUpdate(namespace = OtherNamespace, toUpsert = keyList.zip(valList1))) val valList2 = keyList.map(2.toByte +: _) - db.update(OtherNamespace2, Seq(), keyList.zip(valList2)) + db.update(prepareUpdate(namespace = OtherNamespace2, toUpsert = keyList.zip(valList2))) //Removal of keys from the OtherNamespace namespace - db.update(OtherNamespace, keyList, Nil) + db.update(prepareUpdate(namespace = OtherNamespace, toRemove = keyList)) keyList.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) } - keyList.zip(valList2).foreach { case (key, value) => - assert(db.get(OtherNamespace2, key).contains(value)) + keyList.zip(valList2).foreach { + case (key, value) => + assert(db.get(OtherNamespace2, key).contains(value)) } //Removal of keys from the OtherNamespace2 namespace - db.update(OtherNamespace2, keyList, Nil) + db.update(prepareUpdate(namespace = OtherNamespace2, toRemove = keyList)) keyList.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) @@ -249,7 +237,6 @@ trait DataSourceIntegrationTestBehavior extends ScalaCheckPropertyChecks with Ob keyList.foreach { key => assert(db.get(OtherNamespace2, key).isEmpty) } - db.destroy() } } diff --git a/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala b/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala index d2019ce2cd..3327fded77 100644 --- a/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala +++ b/src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala @@ -4,7 +4,7 @@ import akka.actor.ActorSystem import akka.util.ByteString import com.typesafe.config.ConfigFactory import io.iohk.ethereum.db.components.Storages.PruningModeComponent -import io.iohk.ethereum.db.components.{SharedRocksDbDataSources, Storages} +import io.iohk.ethereum.db.components.{RocksDbDataSourceComponent, Storages} import io.iohk.ethereum.db.storage.{AppStateStorage, StateStorage} import io.iohk.ethereum.db.storage.NodeStorage.{NodeEncoded, NodeHash} import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation @@ -22,6 +22,7 @@ import io.iohk.ethereum.nodebuilder.{AuthHandshakerBuilder, NodeKeyBuilder, Secu import io.iohk.ethereum.utils.{Config, NodeStatus, ServerStatus} import java.util.concurrent.atomic.AtomicReference +import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import org.bouncycastle.util.encoders.Hex import scala.concurrent.duration._ @@ -59,7 +60,7 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit trait PruningConfig extends PruningModeComponent { override val pruningMode: PruningMode = ArchivePruning } - val storagesInstance = new SharedRocksDbDataSources with PruningConfig with Storages.DefaultStorages + val storagesInstance = new RocksDbDataSourceComponent with PruningConfig with Storages.DefaultStorages val blockchain: Blockchain = new BlockchainMock(genesisHash) @@ -114,15 +115,15 @@ object DumpChainApp extends App with NodeKeyBuilder with SecureRandomBuilder wit override def getMptNodeByHash(hash: ByteString): Option[MptNode] = ??? - override def save(blockHeader: BlockHeader): Unit = ??? + override def storeBlockHeader(blockHeader: BlockHeader): DataSourceBatchUpdate = ??? - override def save(blockHash: ByteString, blockBody: BlockBody): Unit = ??? + override def storeBlockBody(blockHash: ByteString, blockBody: BlockBody): DataSourceBatchUpdate = ??? - override def save(blockHash: ByteString, receipts: Seq[Receipt]): Unit = ??? + override def storeReceipts(blockHash: ByteString, receipts: Seq[Receipt]): DataSourceBatchUpdate = ??? - override def save(hash: ByteString, evmCode: ByteString): Unit = ??? + override def storeEvmCode(hash: ByteString, evmCode: ByteString): DataSourceBatchUpdate = ??? - override def save(blockhash: ByteString, totalDifficulty: BigInt): Unit = ??? + override def storeTotalDifficulty(blockhash: ByteString, totalDifficulty: BigInt): DataSourceBatchUpdate = ??? override def saveNode(nodeHash: NodeHash, nodeEncoded: NodeEncoded, blockNumber: BigInt): Unit = ??? diff --git a/src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala b/src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala index f870100662..355b8cb1b6 100644 --- a/src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala +++ b/src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala @@ -3,7 +3,6 @@ package io.iohk.ethereum.txExecTest.util import java.io.Closeable import akka.util.ByteString -import io.iohk.ethereum.db.dataSource.EphemDataSource import io.iohk.ethereum.db.storage._ import io.iohk.ethereum.domain._ import io.iohk.ethereum.domain.BlockHeader._ @@ -12,6 +11,7 @@ import io.iohk.ethereum.network.p2p.messages.PV63._ import MptNodeEncoders._ import ReceiptImplicits._ import io.iohk.ethereum.db.cache.{AppCaches, LruCache} +import io.iohk.ethereum.db.components.EphemDataSourceComponent import io.iohk.ethereum.db.storage.NodeStorage.NodeHash import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode} import io.iohk.ethereum.mpt.{BranchNode, ExtensionNode, HashNode, LeafNode, MptNode} @@ -37,19 +37,19 @@ object FixtureProvider { // scalastyle:off def prepareStorages(blockNumber: BigInt, fixtures: Fixture): BlockchainStorages = { - val storages: BlockchainStorages = new BlockchainStorages with AppCaches { + val storages: BlockchainStorages = new BlockchainStorages with AppCaches with EphemDataSourceComponent { - override val receiptStorage: ReceiptStorage = new ReceiptStorage(EphemDataSource()) - override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(EphemDataSource()) - override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(EphemDataSource()) - override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(EphemDataSource()) - override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(EphemDataSource()) - override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(EphemDataSource()) - override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(EphemDataSource()) - override val nodeStorage: NodeStorage = new NodeStorage(EphemDataSource()) + override val receiptStorage: ReceiptStorage = new ReceiptStorage(dataSource) + override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(dataSource) + override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(dataSource) + override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSource) + override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSource) + override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(dataSource) + override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSource) + override val nodeStorage: NodeStorage = new NodeStorage(dataSource) override val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache) override val pruningMode: PruningMode = ArchivePruning - override val appStateStorage: AppStateStorage = new AppStateStorage(EphemDataSource()) + override val appStateStorage: AppStateStorage = new AppStateStorage(dataSource) override val stateStorage: StateStorage = StateStorage( pruningMode, @@ -63,10 +63,14 @@ object FixtureProvider { val blockchain = BlockchainImpl(storages) blocksToInclude.foreach { case (_, block) => + val receiptsUpdates = fixtures.receipts.get(block.header.hash) + .map(r => storages.receiptStorage.put(block.header.hash, r)) + .getOrElse(storages.receiptStorage.emptyBatchUpdate) storages.blockBodiesStorage.put(block.header.hash, fixtures.blockBodies(block.header.hash)) - storages.blockHeadersStorage.put(block.header.hash, fixtures.blockHeaders(block.header.hash)) - storages.blockNumberMappingStorage.put(block.header.number, block.header.hash) - fixtures.receipts.get(block.header.hash).foreach(r => storages.receiptStorage.put(block.header.hash, r)) + .and(storages.blockHeadersStorage.put(block.header.hash, fixtures.blockHeaders(block.header.hash))) + .and(storages.blockNumberMappingStorage.put(block.header.number, block.header.hash)) + .and(receiptsUpdates) + .commit() def traverse(nodeHash: ByteString): Unit = fixtures.stateMpt.get(nodeHash).orElse(fixtures.contractMpts.get(nodeHash)) match { case Some(m: BranchNode) => @@ -85,7 +89,7 @@ object FixtureProvider { storages.stateStorage.saveNode(ByteString(m.hash), m.toBytes, block.header.number) Try(m.value.toArray[Byte].toAccount).toOption.foreach { account => if (account.codeHash != DumpChainActor.emptyEvm) { - storages.evmCodeStorage.put(account.codeHash, fixtures.evmCode(account.codeHash)) + storages.evmCodeStorage.put(account.codeHash, fixtures.evmCode(account.codeHash)).commit() } if (account.storageRoot != DumpChainActor.emptyStorage) { traverse(account.storageRoot) diff --git a/src/main/scala/io/iohk/ethereum/blockchain/sync/FastSync.scala b/src/main/scala/io/iohk/ethereum/blockchain/sync/FastSync.scala index ed4f35f4ca..1a33257c23 100644 --- a/src/main/scala/io/iohk/ethereum/blockchain/sync/FastSync.scala +++ b/src/main/scala/io/iohk/ethereum/blockchain/sync/FastSync.scala @@ -85,7 +85,7 @@ class FastSync( case FastSyncTargetBlockSelector.Result(targetBlockHeader) => if (targetBlockHeader.number < 1) { log.info("Unable to start block synchronization in fast mode: target block is less than 1") - appStateStorage.fastSyncDone() + appStateStorage.fastSyncDone().commit() context become idle syncController ! Done } else { @@ -230,7 +230,7 @@ class FastSync( blockchain.removeBlock(headerToRemove.hash, withState = false) } } - appStateStorage.putBestBlockNumber((startBlock - blocksToDiscard - 1) max 0) + appStateStorage.putBestBlockNumber((startBlock - blocksToDiscard - 1) max 0).commit() } @tailrec @@ -270,8 +270,9 @@ class FastSync( } private def updateSyncState(header: BlockHeader, parentTd: BigInt): Unit = { - blockchain.save(header) - blockchain.save(header.hash, parentTd + header.difficulty) + blockchain.storeBlockHeader(header) + .and(blockchain.storeTotalDifficulty(header.hash, parentTd + header.difficulty)) + .commit() if (header.number > syncState.bestBlockHeaderNumber) { syncState = syncState.copy(bestBlockHeaderNumber = header.number) @@ -357,9 +358,10 @@ class FastSync( private def handleReceipts(peer: Peer, requestedHashes: Seq[ByteString], receipts: Seq[Seq[Receipt]]) = { validateReceipts(requestedHashes, receipts) match { case ReceiptsValidationResult.Valid(blockHashesWithReceipts) => - blockHashesWithReceipts.foreach { case (hash, receiptsForBlock) => - blockchain.save(hash, receiptsForBlock) - } + blockHashesWithReceipts.map { case (hash, receiptsForBlock) => + blockchain.storeReceipts(hash, receiptsForBlock) + }.reduce(_.and(_)) + .commit() val receivedHashes = blockHashesWithReceipts.unzip._1 updateBestBlockIfNeeded(receivedHashes) @@ -410,7 +412,7 @@ class FastSync( case EvmCodeHash(hash) => val evmCode = nodeData.values(idx) - blockchain.save(hash, evmCode) + blockchain.storeEvmCode(hash, evmCode).commit() Nil case StorageRootHash(_) => @@ -552,9 +554,10 @@ class FastSync( } private def insertBlocks(requestedHashes: Seq[ByteString], blockBodies: Seq[BlockBody]): Unit = { - (requestedHashes zip blockBodies).foreach { case (hash, body) => - blockchain.save(hash, body) - } + (requestedHashes zip blockBodies).map { case (hash, body) => + blockchain.storeBlockBody(hash, body) + }.reduce(_.and(_)) + .commit() val receivedHashes = requestedHashes.take(blockBodies.size) updateBestBlockIfNeeded(receivedHashes) @@ -578,7 +581,7 @@ class FastSync( // We have downloaded to target + fastSyncBlockValidationX, se we must discard those last blocks discardLastBlocks(syncState.safeDownloadTarget, syncConfig.fastSyncBlockValidationX - 1) cleanup() - appStateStorage.fastSyncDone() + appStateStorage.fastSyncDone().commit() context become idle peerRequestsTime = Map.empty syncController ! Done @@ -731,7 +734,7 @@ class FastSync( if (fullBlocks.nonEmpty) { val bestReceivedBlock = fullBlocks.maxBy(_.number) if (appStateStorage.getBestBlockNumber() < bestReceivedBlock.number) { - appStateStorage.putBestBlockNumber(bestReceivedBlock.number) + appStateStorage.putBestBlockNumber(bestReceivedBlock.number).commit() } } diff --git a/src/main/scala/io/iohk/ethereum/db/components/DataSourceComponent.scala b/src/main/scala/io/iohk/ethereum/db/components/DataSourceComponent.scala new file mode 100644 index 0000000000..d973ae5d55 --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/db/components/DataSourceComponent.scala @@ -0,0 +1,7 @@ +package io.iohk.ethereum.db.components + +import io.iohk.ethereum.db.dataSource.DataSource + +trait DataSourceComponent { + val dataSource: DataSource +} diff --git a/src/main/scala/io/iohk/ethereum/db/components/DataSourcesComponent.scala b/src/main/scala/io/iohk/ethereum/db/components/DataSourcesComponent.scala deleted file mode 100644 index 0ae34b24bd..0000000000 --- a/src/main/scala/io/iohk/ethereum/db/components/DataSourcesComponent.scala +++ /dev/null @@ -1,37 +0,0 @@ -package io.iohk.ethereum.db.components - -import io.iohk.ethereum.db.dataSource.DataSource - -trait DataSourcesComponent { - - val dataSources: DataSources - - trait DataSources { - - val evmCodeDataSource: DataSource - - val mptDataSource: DataSource - - val receiptsDataSource: DataSource - - val blockHeadersDataSource: DataSource - - val blockBodiesDataSource: DataSource - - val blockHeightsHashesDataSource: DataSource - - val totalDifficultyDataSource: DataSource - - val appStateDataSource: DataSource - - val fastSyncStateDataSource: DataSource - - val transactionMappingDataSource: DataSource - - val knownNodesDataSource: DataSource - - def closeAll(): Unit - - } - -} diff --git a/src/main/scala/io/iohk/ethereum/db/components/EphemDataSourceComponent.scala b/src/main/scala/io/iohk/ethereum/db/components/EphemDataSourceComponent.scala new file mode 100644 index 0000000000..1466a64c40 --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/db/components/EphemDataSourceComponent.scala @@ -0,0 +1,7 @@ +package io.iohk.ethereum.db.components + +import io.iohk.ethereum.db.dataSource.EphemDataSource + +trait EphemDataSourceComponent extends DataSourceComponent { + val dataSource = EphemDataSource() +} diff --git a/src/main/scala/io/iohk/ethereum/db/components/RocksDbDataSourceComponent.scala b/src/main/scala/io/iohk/ethereum/db/components/RocksDbDataSourceComponent.scala new file mode 100644 index 0000000000..cd4d71a60a --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/db/components/RocksDbDataSourceComponent.scala @@ -0,0 +1,11 @@ +package io.iohk.ethereum.db.components + +import io.iohk.ethereum.db.dataSource.RocksDbDataSource +import io.iohk.ethereum.db.storage.Namespaces +import io.iohk.ethereum.utils.Config + +trait RocksDbDataSourceComponent extends DataSourceComponent { + + lazy val dataSource = RocksDbDataSource(Config.Db.RocksDb, Namespaces.nsSeq) + +} diff --git a/src/main/scala/io/iohk/ethereum/db/components/SharedEphemDataSources.scala b/src/main/scala/io/iohk/ethereum/db/components/SharedEphemDataSources.scala deleted file mode 100644 index 3b637edebc..0000000000 --- a/src/main/scala/io/iohk/ethereum/db/components/SharedEphemDataSources.scala +++ /dev/null @@ -1,35 +0,0 @@ -package io.iohk.ethereum.db.components -import io.iohk.ethereum.db.dataSource.{DataSource, EphemDataSource} - -trait SharedEphemDataSources extends DataSourcesComponent { - - val ephemDataSource = EphemDataSource() - - val dataSources = new DataSources { - - override val evmCodeDataSource: DataSource = ephemDataSource - - override val mptDataSource: DataSource = ephemDataSource - - override val fastSyncStateDataSource: DataSource = ephemDataSource - - override val receiptsDataSource: DataSource = ephemDataSource - - override val blockBodiesDataSource: DataSource = ephemDataSource - - override val blockHeightsHashesDataSource: DataSource = ephemDataSource - - override val blockHeadersDataSource: DataSource = ephemDataSource - - override val totalDifficultyDataSource: DataSource = ephemDataSource - - override val appStateDataSource: DataSource = ephemDataSource - - override val transactionMappingDataSource: DataSource = ephemDataSource - - override val knownNodesDataSource: DataSource = ephemDataSource - - override def closeAll(): Unit = () - } - -} diff --git a/src/main/scala/io/iohk/ethereum/db/components/SharedRocksDbDataSources.scala b/src/main/scala/io/iohk/ethereum/db/components/SharedRocksDbDataSources.scala deleted file mode 100644 index 03633bbcd9..0000000000 --- a/src/main/scala/io/iohk/ethereum/db/components/SharedRocksDbDataSources.scala +++ /dev/null @@ -1,37 +0,0 @@ -package io.iohk.ethereum.db.components - -import io.iohk.ethereum.db.dataSource.{DataSource, RocksDbDataSource} -import io.iohk.ethereum.db.storage.Namespaces -import io.iohk.ethereum.utils.Config - -trait SharedRocksDbDataSources extends DataSourcesComponent { - - lazy val dataSource = RocksDbDataSource(Config.Db.RocksDb, Namespaces.nsSeq) - - lazy val dataSources = new DataSources { - - override val blockBodiesDataSource: DataSource = dataSource - - override val blockHeightsHashesDataSource: DataSource = dataSource - - override val blockHeadersDataSource: DataSource = dataSource - - override val evmCodeDataSource: DataSource = dataSource - - override val mptDataSource: DataSource = dataSource - - override val fastSyncStateDataSource: DataSource = dataSource - - override val receiptsDataSource: DataSource = dataSource - - override val totalDifficultyDataSource: DataSource = dataSource - - override val appStateDataSource: DataSource = dataSource - - override val transactionMappingDataSource: DataSource = dataSource - - override val knownNodesDataSource: DataSource = dataSource - - override def closeAll(): Unit = dataSource.close() - } -} diff --git a/src/main/scala/io/iohk/ethereum/db/components/Storages.scala b/src/main/scala/io/iohk/ethereum/db/components/Storages.scala index d5036f170c..a96ca58b3a 100644 --- a/src/main/scala/io/iohk/ethereum/db/components/Storages.scala +++ b/src/main/scala/io/iohk/ethereum/db/components/Storages.scala @@ -14,36 +14,36 @@ object Storages { trait DefaultStorages extends StoragesComponent { - dataSourcesComp: DataSourcesComponent with PruningModeComponent => + dataSourcesComp: DataSourceComponent with PruningModeComponent => override val storages: Storages = new DefaultStorages(pruningMode) class DefaultStorages(override val pruningMode: PruningMode) extends Storages with AppCaches { - override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(dataSources.blockHeadersDataSource) + override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(dataSource) - override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSources.blockBodiesDataSource) + override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSource) - override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSources.blockHeightsHashesDataSource) + override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSource) - override val receiptStorage: ReceiptStorage = new ReceiptStorage(dataSources.receiptsDataSource) + override val receiptStorage: ReceiptStorage = new ReceiptStorage(dataSource) - override val nodeStorage: NodeStorage = new NodeStorage(dataSources.mptDataSource) + override val nodeStorage: NodeStorage = new NodeStorage(dataSource) override val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache) - override val fastSyncStateStorage: FastSyncStateStorage = new FastSyncStateStorage(dataSources.fastSyncStateDataSource) + override val fastSyncStateStorage: FastSyncStateStorage = new FastSyncStateStorage(dataSource) - override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(dataSources.evmCodeDataSource) + override val evmCodeStorage: EvmCodeStorage = new EvmCodeStorage(dataSource) override val totalDifficultyStorage: TotalDifficultyStorage = - new TotalDifficultyStorage(dataSources.totalDifficultyDataSource) + new TotalDifficultyStorage(dataSource) - override val appStateStorage: AppStateStorage = new AppStateStorage(dataSources.appStateDataSource) + override val appStateStorage: AppStateStorage = new AppStateStorage(dataSource) - override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSources.transactionMappingDataSource) + override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSource) - override val knownNodesStorage: KnownNodesStorage = new KnownNodesStorage(dataSources.knownNodesDataSource) + override val knownNodesStorage: KnownNodesStorage = new KnownNodesStorage(dataSource) override val stateStorage: StateStorage = StateStorage( diff --git a/src/main/scala/io/iohk/ethereum/db/dataSource/DataSource.scala b/src/main/scala/io/iohk/ethereum/db/dataSource/DataSource.scala index cb62bb78e2..fbfb5a8712 100644 --- a/src/main/scala/io/iohk/ethereum/db/dataSource/DataSource.scala +++ b/src/main/scala/io/iohk/ethereum/db/dataSource/DataSource.scala @@ -33,33 +33,14 @@ trait DataSource { /** * This function updates the DataSource by deleting, updating and inserting new (key-value) pairs. - * - * @param namespace from which the (key-value) pairs will be removed and inserted. - * @param toRemove which includes all the keys to be removed from the DataSource. - * @param toUpsert which includes all the (key-value) pairs to be inserted into the DataSource. - * If a key is already in the DataSource its value will be updated. - * @return the new DataSource after the removals and insertions were done. + * Implementations should guarantee that the whole operation is atomic. */ - def update(namespace: Namespace, toRemove: Seq[Key], toUpsert: Seq[(Key, Value)]): DataSource - - /** - * This function updates the DataSource by deleting, updating and inserting new (key-value) pairs. - * It assumes that caller already properly serialized key and value. - * Useful when caller knows some pattern in data to avoid generic serialization. - * - * @param toRemove which includes all the keys to be removed from the DataSource. - * @param toUpsert which includes all the (key-value) pairs to be inserted into the DataSource. - * If a key is already in the DataSource its value will be updated. - * @return the new DataSource after the removals and insertions were done. - */ - def updateOptimized(toRemove: Seq[Array[Byte]], toUpsert: Seq[(Array[Byte], Array[Byte])]): DataSource + def update(dataSourceUpdates: Seq[DataUpdate]): Unit /** * This function updates the DataSource by deleting all the (key-value) pairs in it. - * - * @return the new DataSource after all the data was removed. */ - def clear: DataSource + def clear(): Unit /** * This function closes the DataSource, without deleting the files used by it. diff --git a/src/main/scala/io/iohk/ethereum/db/dataSource/DataSourceBatchUpdate.scala b/src/main/scala/io/iohk/ethereum/db/dataSource/DataSourceBatchUpdate.scala new file mode 100644 index 0000000000..da47a2c313 --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/db/dataSource/DataSourceBatchUpdate.scala @@ -0,0 +1,17 @@ +package io.iohk.ethereum.db.dataSource + +case class DataSourceBatchUpdate(dataSource: DataSource, updates: Array[DataUpdate] = Array.empty) { + + def and(that: DataSourceBatchUpdate): DataSourceBatchUpdate = { + require( + this.dataSource eq that.dataSource, + "Transactional storage updates must be performed on the same data source" + ) + DataSourceBatchUpdate(dataSource, this.updates ++ that.updates) + } + + def commit(): Unit = { + dataSource.update(updates) + } + +} diff --git a/src/main/scala/io/iohk/ethereum/db/dataSource/DataSourceUpdate.scala b/src/main/scala/io/iohk/ethereum/db/dataSource/DataSourceUpdate.scala new file mode 100644 index 0000000000..123d52997a --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/db/dataSource/DataSourceUpdate.scala @@ -0,0 +1,26 @@ +package io.iohk.ethereum.db.dataSource + +import io.iohk.ethereum.db.dataSource.DataSource.{Key, Namespace, Value} + +sealed trait DataUpdate + +/** + * This represent updates to be performed on the DataSource by deleting, updating and inserting new (key-value) pairs. + * + * @param namespace from which the (key-value) pairs will be removed and inserted. + * @param toRemove which includes all the keys to be removed from the DataSource. + * @param toUpsert which includes all the (key-value) pairs to be inserted into the DataSource. + * If a key is already in the DataSource its value will be updated. + */ +case class DataSourceUpdate(namespace: Namespace, toRemove: Seq[Key], toUpsert: Seq[(Key, Value)]) extends DataUpdate + +/** + * This represent updates the DataSource by deleting, updating and inserting new (key-value) pairs. + * It assumes that caller already properly serialized key and value. + * Useful when caller knows some pattern in data to avoid generic serialization. + * + * @param toRemove which includes all the keys to be removed from the DataSource. + * @param toUpsert which includes all the (key-value) pairs to be inserted into the DataSource. + * If a key is already in the DataSource its value will be updated. + */ +case class DataSourceUpdateOptimized(toRemove: Seq[Array[Byte]], toUpsert: Seq[(Array[Byte], Array[Byte])]) extends DataUpdate diff --git a/src/main/scala/io/iohk/ethereum/db/dataSource/EphemDataSource.scala b/src/main/scala/io/iohk/ethereum/db/dataSource/EphemDataSource.scala index ad06c8af60..7754d3d16f 100644 --- a/src/main/scala/io/iohk/ethereum/db/dataSource/EphemDataSource.scala +++ b/src/main/scala/io/iohk/ethereum/db/dataSource/EphemDataSource.scala @@ -9,38 +9,48 @@ class EphemDataSource(var storage: Map[ByteBuffer, Array[Byte]]) extends DataSou * key.drop to remove namespace prefix from the key * @return key values paris from this storage */ - def getAll(namespace: Namespace): Seq[(IndexedSeq[Byte], IndexedSeq[Byte])] = - storage.toSeq.map{case (key, value) => (key.array().drop(namespace.length).toIndexedSeq, value.toIndexedSeq)} - - override def get(namespace: Namespace, key: Key): Option[Value] = storage.get(ByteBuffer.wrap((namespace ++ key).toArray)).map(_.toIndexedSeq) - - override def update(namespace: Namespace, toRemove: Seq[Key], toUpsert: Seq[(Key, Value)]): DataSource = { - val afterRemoval = toRemove.foldLeft(storage)((storage, key) => storage - ByteBuffer.wrap((namespace ++ key).toArray)) - val afterUpdate = toUpsert.foldLeft(afterRemoval)((storage, toUpdate) => - storage + (ByteBuffer.wrap((namespace ++ toUpdate._1).toArray) -> toUpdate._2.toArray)) - storage = afterUpdate - this + def getAll(namespace: Namespace): Seq[(IndexedSeq[Byte], IndexedSeq[Byte])] = synchronized { + storage.toSeq.map { case (key, value) => (key.array().drop(namespace.length).toIndexedSeq, value.toIndexedSeq) } } - override def clear: DataSource = { - storage = Map() - this + override def get(namespace: Namespace, key: Key): Option[Value] = { + storage.get(ByteBuffer.wrap((namespace ++ key).toArray)).map(_.toIndexedSeq) } - override def close(): Unit = () + override def getOptimized(key: Array[Byte]): Option[Array[Byte]] = storage.get(ByteBuffer.wrap(key)) - override def destroy(): Unit = () + override def update(dataSourceUpdates: Seq[DataUpdate]): Unit = synchronized { + dataSourceUpdates.foreach { + case DataSourceUpdate(namespace, toRemove, toUpsert) => + update(namespace, toRemove, toUpsert) + case DataSourceUpdateOptimized(toRemove, toUpsert) => + updateOptimized(toRemove, toUpsert) + } + } - override def updateOptimized(toRemove: Seq[Array[Byte]], toUpsert: Seq[(Array[Byte], Array[Byte])]): DataSource = { + private def update(namespace: Namespace, toRemove: Seq[Key], toUpsert: Seq[(Key, Value)]): Unit = synchronized { + val afterRemoval = + toRemove.foldLeft(storage)((storage, key) => storage - ByteBuffer.wrap((namespace ++ key).toArray)) + val afterUpdate = toUpsert.foldLeft(afterRemoval)( + (storage, toUpdate) => storage + (ByteBuffer.wrap((namespace ++ toUpdate._1).toArray) -> toUpdate._2.toArray) + ) + storage = afterUpdate + } + + private def updateOptimized(toRemove: Seq[Array[Byte]], toUpsert: Seq[(Array[Byte], Array[Byte])]): Unit = synchronized { val afterRemoval = toRemove.foldLeft(storage)((storage, key) => storage - ByteBuffer.wrap(key)) val afterUpdate = toUpsert.foldLeft(afterRemoval)((storage, toUpdate) => storage + (ByteBuffer.wrap(toUpdate._1) -> toUpdate._2)) storage = afterUpdate - this + } + override def clear(): Unit = synchronized { + storage = Map() } - override def getOptimized(key: Array[Byte]): Option[Array[Byte]] = storage.get(ByteBuffer.wrap(key)) + override def close(): Unit = () + + override def destroy(): Unit = () } object EphemDataSource { diff --git a/src/main/scala/io/iohk/ethereum/db/dataSource/RocksDbDataSource.scala b/src/main/scala/io/iohk/ethereum/db/dataSource/RocksDbDataSource.scala index 196880fb7f..215be693f1 100644 --- a/src/main/scala/io/iohk/ethereum/db/dataSource/RocksDbDataSource.scala +++ b/src/main/scala/io/iohk/ethereum/db/dataSource/RocksDbDataSource.scala @@ -68,75 +68,44 @@ class RocksDbDataSource( } } - /** - * This function updates the DataSource by deleting, updating and inserting new (key-value) pairs. - * - * @param namespace from which the (key-value) pairs will be removed and inserted. - * @param toRemove which includes all the keys to be removed from the DataSource. - * @param toUpsert which includes all the (key-value) pairs to be inserted into the DataSource. - * If a key is already in the DataSource its value will be updated. - * @return the new DataSource after the removals and insertions were done. - */ - override def update(namespace: Namespace, toRemove: Seq[Key], toUpsert: Seq[(Key, Value)]): DataSource = { + override def update(dataSourceUpdates: Seq[DataUpdate]): Unit = { assureNotClosed() RocksDbDataSource.dbLock.readLock().lock() try { - withResources(new WriteOptions()){ writeOptions => - withResources(new WriteBatch()){ batch => - toRemove.foreach{ key => batch.delete(handles(namespace), key.toArray) } - toUpsert.foreach{ case (k, v) => batch.put(handles(namespace), k.toArray, v.toArray) } - + withResources(new WriteOptions()) { writeOptions => + withResources(new WriteBatch()) { batch => + dataSourceUpdates.foreach { + case DataSourceUpdate(namespace, toRemove, toUpsert) => + toRemove.foreach { key => + batch.delete(handles(namespace), key.toArray) + } + toUpsert.foreach { case (k, v) => batch.put(handles(namespace), k.toArray, v.toArray) } + + case DataSourceUpdateOptimized(toRemove, toUpsert) => + toRemove.foreach { key => + batch.delete(key) + } + toUpsert.foreach { case (k, v) => batch.put(k, v) } + } db.write(writeOptions, batch) } } } catch { case NonFatal(e) => - logger.error(s"DataSource not updated (toRemove: ${ toRemove.size }, toUpsert: ${ toUpsert.size }, namespace: $namespace), cause: {}", e.getMessage) + logger.error( + s"DataSource not updated, cause: {}", + e.getMessage + ) throw new RuntimeException(e) } finally { RocksDbDataSource.dbLock.readLock().unlock() } - this - } - - /** - * This function updates the DataSource by deleting, updating and inserting new (key-value) pairs. - * It assumes that caller already properly serialized key and value. - * Useful when caller knows some pattern in data to avoid generic serialization. - * - * @param toRemove which includes all the keys to be removed from the DataSource. - * @param toUpsert which includes all the (key-value) pairs to be inserted into the DataSource. - * If a key is already in the DataSource its value will be updated. - * @return the new DataSource after the removals and insertions were done. - */ - override def updateOptimized(toRemove: Seq[Array[Byte]], toUpsert: Seq[(Array[Byte], Array[Byte])]): DataSource = { - assureNotClosed() - RocksDbDataSource.dbLock.readLock().lock() - try { - withResources(new WriteOptions()){ writeOptions => - withResources(new WriteBatch()){ batch => - toRemove.foreach{ key => batch.delete(key) } - toUpsert.foreach{ case (k, v) => batch.put(k, v) } - - db.write(writeOptions, batch) - } - } - } catch { - case NonFatal(e) => - logger.error(s"DataSource not updated (toRemove: ${ toRemove.size }, toUpsert: ${ toUpsert.size }), cause: {}", e.getMessage) - throw new RuntimeException(e) - } finally { - RocksDbDataSource.dbLock.readLock().unlock() - } - this } /** * This function updates the DataSource by deleting all the (key-value) pairs in it. - * - * @return the new DataSource after all the data was removed. */ - override def clear: DataSource = { + override def clear(): Unit = { destroy() logger.debug(s"About to create new DataSource for path: ${ rocksDbConfig.path }") val (newDb, handles, readOptions, dbOptions, cfOptions) = RocksDbDataSource.createDB(rocksDbConfig, nameSpaces.tail) @@ -149,7 +118,6 @@ class RocksDbDataSource( this.dbOptions = dbOptions this.cfOptions = cfOptions this.isClosed = false - this } /** diff --git a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala index 3465d22781..a05d6fba16 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/AppStateStorage.scala @@ -1,6 +1,6 @@ package io.iohk.ethereum.db.storage -import io.iohk.ethereum.db.dataSource.DataSource +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceBatchUpdate} import io.iohk.ethereum.db.storage.AppStateStorage._ /** @@ -8,7 +8,7 @@ import io.iohk.ethereum.db.storage.AppStateStorage._ * Key: see AppStateStorage.Keys * Value: stored string value */ -class AppStateStorage(val dataSource: DataSource) extends KeyValueStorage[Key, Value, AppStateStorage]{ +class AppStateStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[Key, Value]{ type T = AppStateStorage val namespace: IndexedSeq[Byte] = Namespaces.AppStateNamespace @@ -21,25 +21,25 @@ class AppStateStorage(val dataSource: DataSource) extends KeyValueStorage[Key, V def getBestBlockNumber(): BigInt = BigInt(get(Keys.BestBlockNumber).getOrElse("0")) - def putBestBlockNumber(bestBlockNumber: BigInt): AppStateStorage = + def putBestBlockNumber(bestBlockNumber: BigInt): DataSourceBatchUpdate = put(Keys.BestBlockNumber, bestBlockNumber.toString) def isFastSyncDone(): Boolean = get(Keys.FastSyncDone).exists(_.toBoolean) - def fastSyncDone(): AppStateStorage = + def fastSyncDone(): DataSourceBatchUpdate = put(Keys.FastSyncDone, true.toString) def getEstimatedHighestBlock(): BigInt = BigInt(get(Keys.EstimatedHighestBlock).getOrElse("0")) - def putEstimatedHighestBlock(n: BigInt): AppStateStorage = + def putEstimatedHighestBlock(n: BigInt): DataSourceBatchUpdate = put(Keys.EstimatedHighestBlock, n.toString) def getSyncStartingBlock(): BigInt = BigInt(get(Keys.SyncStartingBlock).getOrElse("0")) - def putSyncStartingBlock(n: BigInt): AppStateStorage = + def putSyncStartingBlock(n: BigInt): DataSourceBatchUpdate = put(Keys.SyncStartingBlock, n.toString) } diff --git a/src/main/scala/io/iohk/ethereum/db/storage/BlockBodiesStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/BlockBodiesStorage.scala index 77cb2a5ffc..f6a67f4896 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/BlockBodiesStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/BlockBodiesStorage.scala @@ -15,7 +15,7 @@ import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes * Key: hash of the block to which the BlockBody belong * Value: the block body */ -class BlockBodiesStorage(val dataSource: DataSource) extends KeyValueStorage[BlockBodyHash, BlockBody, BlockBodiesStorage] { +class BlockBodiesStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[BlockBodyHash, BlockBody] { import BlockBodiesStorage._ override val namespace: IndexedSeq[Byte] = Namespaces.BodyNamespace @@ -26,8 +26,6 @@ class BlockBodiesStorage(val dataSource: DataSource) extends KeyValueStorage[Blo override def valueDeserializer: IndexedSeq[Byte] => BlockBody = bytes => Unpickle[BlockBody].fromBytes(ByteBuffer.wrap(bytes.toArray[Byte])) - - override protected def apply(dataSource: DataSource): BlockBodiesStorage = new BlockBodiesStorage(dataSource) } object BlockBodiesStorage { diff --git a/src/main/scala/io/iohk/ethereum/db/storage/BlockHeadersStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/BlockHeadersStorage.scala index c160a0c2d0..20123a7137 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/BlockHeadersStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/BlockHeadersStorage.scala @@ -14,7 +14,7 @@ import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes * Key: hash of the block to which the BlockHeader belong * Value: the block header */ -class BlockHeadersStorage(val dataSource: DataSource) extends KeyValueStorage[BlockHeaderHash, BlockHeader, BlockHeadersStorage] { +class BlockHeadersStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[BlockHeaderHash, BlockHeader] { import BlockHeadersStorage._ @@ -27,9 +27,6 @@ class BlockHeadersStorage(val dataSource: DataSource) extends KeyValueStorage[Bl override def valueDeserializer: IndexedSeq[Byte] => BlockHeader = bytes => Unpickle[BlockHeader].fromBytes(ByteBuffer.wrap(bytes.toArray[Byte])) - - override protected def apply(dataSource: DataSource): BlockHeadersStorage = new BlockHeadersStorage(dataSource) - } object BlockHeadersStorage { diff --git a/src/main/scala/io/iohk/ethereum/db/storage/BlockNumberMappingStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/BlockNumberMappingStorage.scala index dab9ab9b0d..121d46b983 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/BlockNumberMappingStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/BlockNumberMappingStorage.scala @@ -4,7 +4,7 @@ import akka.util.ByteString import io.iohk.ethereum.db.dataSource.DataSource import io.iohk.ethereum.db.storage.BlockHeadersStorage.BlockHeaderHash -class BlockNumberMappingStorage(val dataSource: DataSource) extends KeyValueStorage[BigInt, BlockHeaderHash, BlockNumberMappingStorage] { +class BlockNumberMappingStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[BigInt, BlockHeaderHash] { override val namespace: IndexedSeq[Byte] = Namespaces.HeightsNamespace override def keySerializer: (BigInt) => IndexedSeq[Byte] = index => index.toByteArray @@ -12,6 +12,4 @@ class BlockNumberMappingStorage(val dataSource: DataSource) extends KeyValueStor override def valueSerializer: (BlockHeaderHash) => IndexedSeq[Byte] = identity override def valueDeserializer: (IndexedSeq[Byte]) => BlockHeaderHash = arr => ByteString(arr.toArray[Byte]) - - override protected def apply(dataSource: DataSource): BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSource) } diff --git a/src/main/scala/io/iohk/ethereum/db/storage/CachedKeyValueStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/CachedKeyValueStorage.scala index 805cf4702d..f93958f39f 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/CachedKeyValueStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/CachedKeyValueStorage.scala @@ -28,7 +28,7 @@ trait CachedKeyValueStorage[K, V, T <: CachedKeyValueStorage[K, V, T]] extends S def forcePersist(): Unit = { storage.update(Nil, cache.getValues) - cache.clear + cache.clear() } // TODO EC-491 Consider other persist strategy like sliding window (save and clear only old stuff which survived long enough) @@ -41,4 +41,3 @@ trait CachedKeyValueStorage[K, V, T <: CachedKeyValueStorage[K, V, T]] extends S } } } - diff --git a/src/main/scala/io/iohk/ethereum/db/storage/EvmCodeStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/EvmCodeStorage.scala index eb38ba9742..538e7ebc1e 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/EvmCodeStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/EvmCodeStorage.scala @@ -9,13 +9,11 @@ import io.iohk.ethereum.db.storage.EvmCodeStorage._ * Key: hash of the code * Value: the code */ -class EvmCodeStorage(val dataSource: DataSource) extends KeyValueStorage[CodeHash, Code, EvmCodeStorage] { +class EvmCodeStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[CodeHash, Code] { val namespace: IndexedSeq[Byte] = Namespaces.CodeNamespace def keySerializer: CodeHash => IndexedSeq[Byte] = identity def valueSerializer: Code => IndexedSeq[Byte] = identity def valueDeserializer: IndexedSeq[Byte] => Code = (code: IndexedSeq[Byte]) => ByteString(code.toArray) - - protected def apply(dataSource: DataSource): EvmCodeStorage = new EvmCodeStorage(dataSource) } object EvmCodeStorage { diff --git a/src/main/scala/io/iohk/ethereum/db/storage/KeyValueStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/KeyValueStorage.scala index f5355962bd..594cfed6db 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/KeyValueStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/KeyValueStorage.scala @@ -1,9 +1,9 @@ package io.iohk.ethereum.db.storage import io.iohk.ethereum.common.SimpleMap -import io.iohk.ethereum.db.dataSource.DataSource +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdate} -trait KeyValueStorage[K, V, T <: KeyValueStorage[K, V, T]] extends SimpleMap[K, V, T]{ +trait KeyValueStorage[K, V, T <: KeyValueStorage[K, V, T]] extends SimpleMap[K, V, T] { val dataSource: DataSource val namespace: IndexedSeq[Byte] @@ -31,39 +31,12 @@ trait KeyValueStorage[K, V, T <: KeyValueStorage[K, V, T]] extends SimpleMap[K, * @return the new KeyValueStorage after the removals and insertions were done. */ def update(toRemove: Seq[K], toUpsert: Seq[(K, V)]): T = { - val newDataSource = dataSource.update( - namespace = namespace, - toRemove = toRemove.map(keySerializer), - toUpsert = toUpsert.map { case (k, v) => keySerializer(k) -> valueSerializer(v) } - ) - apply(newDataSource) + dataSource.update(Seq( + DataSourceUpdate( + namespace, + toRemove.map(keySerializer), + toUpsert.map { case (k, v) => keySerializer(k) -> valueSerializer(v) }) + )) + apply(dataSource) } } - -object Namespaces { - val ReceiptsNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('r'.toByte) - val HeaderNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('h'.toByte) - val BodyNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('b'.toByte) - val NodeNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('n'.toByte) - val CodeNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('c'.toByte) - val TotalDifficultyNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('t'.toByte) - val AppStateNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('s'.toByte) - val KnownNodesNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('k'.toByte) - val HeightsNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('i'.toByte) - val FastSyncStateNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('f'.toByte) - val TransactionMappingNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('l'.toByte) - - val nsSeq = Seq( - ReceiptsNamespace, - HeaderNamespace, - BodyNamespace, - NodeNamespace, - CodeNamespace, - TotalDifficultyNamespace, - AppStateNamespace, - KnownNodesNamespace, - HeightsNamespace, - FastSyncStateNamespace, - TransactionMappingNamespace - ) -} diff --git a/src/main/scala/io/iohk/ethereum/db/storage/KnownNodesStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/KnownNodesStorage.scala index 48e5a96141..fcabf50b6e 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/KnownNodesStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/KnownNodesStorage.scala @@ -2,15 +2,13 @@ package io.iohk.ethereum.db.storage import java.net.URI -import io.iohk.ethereum.db.dataSource.DataSource +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceBatchUpdate} /** * This class is used to store discovered nodes * Value: stored nodes list */ -class KnownNodesStorage(val dataSource: DataSource) extends KeyValueStorage[String, Set[String], KnownNodesStorage]{ - type T = KnownNodesStorage - +class KnownNodesStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[String, Set[String]]{ val key = "KnownNodes" val namespace: IndexedSeq[Byte] = Namespaces.KnownNodesNamespace @@ -18,13 +16,11 @@ class KnownNodesStorage(val dataSource: DataSource) extends KeyValueStorage[Stri def valueSerializer: Set[String] => IndexedSeq[Byte] = _.mkString(" ").getBytes def valueDeserializer: IndexedSeq[Byte] => Set[String] = (valueBytes: IndexedSeq[Byte]) => new String(valueBytes.toArray).split(' ').toSet - protected def apply(dataSource: DataSource): KnownNodesStorage = new KnownNodesStorage(dataSource) - def getKnownNodes(): Set[URI] = { get(key).getOrElse(Set.empty).filter(_.nonEmpty).map(new URI(_)) } - def updateKnownNodes(toAdd: Set[URI] = Set.empty, toRemove: Set[URI] = Set.empty): KnownNodesStorage = { + def updateKnownNodes(toAdd: Set[URI] = Set.empty, toRemove: Set[URI] = Set.empty): DataSourceBatchUpdate = { val updated = (getKnownNodes() ++ toAdd) -- toRemove put(key, updated.map(_.toString)) } diff --git a/src/main/scala/io/iohk/ethereum/db/storage/Namespaces.scala b/src/main/scala/io/iohk/ethereum/db/storage/Namespaces.scala new file mode 100644 index 0000000000..58b8ebf52c --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/db/storage/Namespaces.scala @@ -0,0 +1,29 @@ +package io.iohk.ethereum.db.storage + +object Namespaces { + val ReceiptsNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('r'.toByte) + val HeaderNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('h'.toByte) + val BodyNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('b'.toByte) + val NodeNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('n'.toByte) + val CodeNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('c'.toByte) + val TotalDifficultyNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('t'.toByte) + val AppStateNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('s'.toByte) + val KnownNodesNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('k'.toByte) + val HeightsNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('i'.toByte) + val FastSyncStateNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('f'.toByte) + val TransactionMappingNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('l'.toByte) + + val nsSeq = Seq( + ReceiptsNamespace, + HeaderNamespace, + BodyNamespace, + NodeNamespace, + CodeNamespace, + TotalDifficultyNamespace, + AppStateNamespace, + KnownNodesNamespace, + HeightsNamespace, + FastSyncStateNamespace, + TransactionMappingNamespace + ) +} diff --git a/src/main/scala/io/iohk/ethereum/db/storage/NodeStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/NodeStorage.scala index ffbb598079..49d2aaf3d9 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/NodeStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/NodeStorage.scala @@ -2,10 +2,9 @@ package io.iohk.ethereum.db.storage import akka.util.ByteString import io.iohk.ethereum.db.cache.Cache -import io.iohk.ethereum.db.dataSource.DataSource +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdateOptimized} import io.iohk.ethereum.db.storage.NodeStorage.{NodeEncoded, NodeHash} - sealed trait NodesStorage extends { def get(key: NodeHash): Option[NodeEncoded] def update(toRemove: Seq[NodeHash], toUpsert: Seq[(NodeHash, NodeEncoded)]): NodesStorage @@ -41,11 +40,15 @@ class NodeStorage(val dataSource: DataSource) extends KeyValueStorage[NodeHash, * @return the new KeyValueStorage after the removals and insertions were done. */ override def update(toRemove: Seq[NodeHash], toUpsert: Seq[(NodeHash, NodeEncoded)]): NodeStorage = { - val newDataSource = dataSource.updateOptimized( - toRemove = toRemove.map(specialSerializer), - toUpsert = toUpsert.map(values => specialSerializer(values._1) -> values._2) + dataSource.update( + Seq( + DataSourceUpdateOptimized( + toRemove = toRemove.map(specialSerializer), + toUpsert = toUpsert.map(values => specialSerializer(values._1) -> values._2) + ) + ) ) - apply(newDataSource) + apply(dataSource) } protected def apply(dataSource: DataSource): NodeStorage = new NodeStorage(dataSource) @@ -61,7 +64,6 @@ class CachedNodeStorage(val storage: NodeStorage, val cache: Cache[NodeHash, Nod override def apply(cache: Cache[NodeHash, NodeEncoded], storage: NodeStorage): CachedNodeStorage = new CachedNodeStorage(storage, cache) } - object NodeStorage { type NodeHash = ByteString type NodeEncoded = Array[Byte] diff --git a/src/main/scala/io/iohk/ethereum/db/storage/ReceiptStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/ReceiptStorage.scala index 5c13af5a02..c494797d77 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/ReceiptStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/ReceiptStorage.scala @@ -15,7 +15,7 @@ import boopickle.DefaultBasic._ * Key: hash of the block to which the list of receipts belong * Value: the list of receipts */ -class ReceiptStorage(val dataSource: DataSource) extends KeyValueStorage[BlockHash, Seq[Receipt], ReceiptStorage] { +class ReceiptStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[BlockHash, Seq[Receipt]] { import ReceiptStorage._ @@ -27,8 +27,6 @@ class ReceiptStorage(val dataSource: DataSource) extends KeyValueStorage[BlockHa override def valueDeserializer: IndexedSeq[Byte] => ReceiptSeq = bytes => Unpickle[Seq[Receipt]].fromBytes(ByteBuffer.wrap(bytes.toArray[Byte])) - - override protected def apply(dataSource: DataSource): ReceiptStorage = new ReceiptStorage(dataSource) } object ReceiptStorage { diff --git a/src/main/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorage.scala index f8a29528d6..984758244c 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorage.scala @@ -188,7 +188,7 @@ object ReferenceCountNodeStorage extends PruneSupport with Logger { * from DB. To do so, it checks if nodes marked in death row have still reference count equal to 0 and are not used by future * blocks. * @param blockNumber - * @param snapshotKeys + * @param deadRowKey * @param nodeStorage * @return */ diff --git a/src/main/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorage.scala index ac2b11ff96..542dc71ce1 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorage.scala @@ -9,13 +9,11 @@ import io.iohk.ethereum.db.storage.TotalDifficultyStorage._ * Key: hash of the block * Value: the total difficulty */ -class TotalDifficultyStorage(val dataSource: DataSource) extends KeyValueStorage[BlockHash, TotalDifficulty, TotalDifficultyStorage]{ +class TotalDifficultyStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[BlockHash, TotalDifficulty]{ val namespace: IndexedSeq[Byte] = Namespaces.TotalDifficultyNamespace def keySerializer: BlockHash => IndexedSeq[Byte] = _.toIndexedSeq def valueSerializer: TotalDifficulty => IndexedSeq[Byte] = _.toByteArray.toIndexedSeq def valueDeserializer: IndexedSeq[Byte] => BigInt = (valueBytes: IndexedSeq[Byte]) => BigInt(1, valueBytes.toArray) - - protected def apply(dataSource: DataSource): TotalDifficultyStorage = new TotalDifficultyStorage(dataSource) } object TotalDifficultyStorage { diff --git a/src/main/scala/io/iohk/ethereum/db/storage/TransactionMappingStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/TransactionMappingStorage.scala index f92fb91b49..6da6a273da 100644 --- a/src/main/scala/io/iohk/ethereum/db/storage/TransactionMappingStorage.scala +++ b/src/main/scala/io/iohk/ethereum/db/storage/TransactionMappingStorage.scala @@ -8,7 +8,7 @@ import io.iohk.ethereum.db.storage.TransactionMappingStorage.{TransactionLocatio import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes import boopickle.Default._ -class TransactionMappingStorage(val dataSource: DataSource) extends KeyValueStorage[TxHash, TransactionLocation, TransactionMappingStorage] { +class TransactionMappingStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[TxHash, TransactionLocation] { val namespace: IndexedSeq[Byte] = Namespaces.TransactionMappingNamespace def keySerializer: TxHash => IndexedSeq[Byte] = identity @@ -17,9 +17,6 @@ class TransactionMappingStorage(val dataSource: DataSource) extends KeyValueStor bytes => Unpickle[TransactionLocation].fromBytes(ByteBuffer.wrap(bytes.toArray[Byte])) implicit val byteStringPickler: Pickler[ByteString] = transformPickler[ByteString, Array[Byte]](ByteString(_))(_.toArray[Byte]) - - protected def apply(dataSource: DataSource): TransactionMappingStorage = new TransactionMappingStorage(dataSource) - } object TransactionMappingStorage { diff --git a/src/main/scala/io/iohk/ethereum/db/storage/TransactionalKeyValueStorage.scala b/src/main/scala/io/iohk/ethereum/db/storage/TransactionalKeyValueStorage.scala new file mode 100644 index 0000000000..a4e813761b --- /dev/null +++ b/src/main/scala/io/iohk/ethereum/db/storage/TransactionalKeyValueStorage.scala @@ -0,0 +1,44 @@ +package io.iohk.ethereum.db.storage + +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceBatchUpdate, DataSourceUpdate} + +/** + * Represents transactional key value storage mapping keys of type K to values of type V + * Note: all methods methods that perform updates return [[io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate]] + * meaning no updates are actually saved in the underlying DataSource until `.commit()` is called. + */ +trait TransactionalKeyValueStorage[K, V] { + + val dataSource: DataSource + val namespace: IndexedSeq[Byte] + def keySerializer: K => IndexedSeq[Byte] + def valueSerializer: V => IndexedSeq[Byte] + def valueDeserializer: IndexedSeq[Byte] => V + + /** + * This function obtains the associated value to a key in the current namespace, if there exists one. + * + * @param key + * @return the value associated with the passed key, if there exists one. + */ + def get(key: K): Option[V] = dataSource.get(namespace, keySerializer(key)).map(valueDeserializer) + + /** + * This function creates a batch of updates to the KeyValueStorage by deleting, updating and inserting new (key-value) + * pairs in the current namespace. The batch should be committed atomically. + */ + def update(toRemove: Seq[K], toUpsert: Seq[(K, V)]): DataSourceBatchUpdate = { + DataSourceBatchUpdate(dataSource, Array(DataSourceUpdate(namespace, toRemove.map(keySerializer), toUpsert.map { + case (k, v) => keySerializer(k) -> valueSerializer(v) + }))) + } + + def put(key: K, value: V): DataSourceBatchUpdate = + update(Nil, Seq(key -> value)) + + def remove(key: K): DataSourceBatchUpdate = + update(Seq(key), Nil) + + def emptyBatchUpdate: DataSourceBatchUpdate = + DataSourceBatchUpdate(dataSource, Array.empty) +} diff --git a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala index 187427b435..8d2784542e 100644 --- a/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala +++ b/src/main/scala/io/iohk/ethereum/domain/Blockchain.scala @@ -3,6 +3,7 @@ package io.iohk.ethereum.domain import java.util.concurrent.atomic.AtomicReference import akka.util.ByteString +import io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate import io.iohk.ethereum.db.storage.NodeStorage.{NodeEncoded, NodeHash} import io.iohk.ethereum.db.storage.StateStorage.RollBackFlush import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation @@ -131,12 +132,13 @@ trait Blockchain { /** * Persists a block in the underlying Blockchain Database + * Note: all store* do not update the database immediately, rather they create + * a [[io.iohk.ethereum.db.dataSource.DataSourceBatchUpdate]] which then has to be committed (atomic operation) * * @param block Block to be saved */ - def save(block: Block): Unit = { - save(block.header) - save(block.header.hash, block.body) + def storeBlock(block: Block): DataSourceBatchUpdate = { + storeBlockHeader(block.header).and(storeBlockBody(block.header.hash, block.body)) } def removeBlock(hash: ByteString, withState: Boolean): Unit @@ -146,15 +148,15 @@ trait Blockchain { * * @param blockHeader Block to be saved */ - def save(blockHeader: BlockHeader): Unit + def storeBlockHeader(blockHeader: BlockHeader): DataSourceBatchUpdate - def save(blockHash: ByteString, blockBody: BlockBody): Unit + def storeBlockBody(blockHash: ByteString, blockBody: BlockBody): DataSourceBatchUpdate - def save(blockHash: ByteString, receipts: Seq[Receipt]): Unit + def storeReceipts(blockHash: ByteString, receipts: Seq[Receipt]): DataSourceBatchUpdate - def save(hash: ByteString, evmCode: ByteString): Unit + def storeEvmCode(hash: ByteString, evmCode: ByteString): DataSourceBatchUpdate - def save(blockhash: ByteString, totalDifficulty: BigInt): Unit + def storeTotalDifficulty(blockhash: ByteString, totalDifficulty: BigInt): DataSourceBatchUpdate def saveBestKnownBlock(number: BigInt): Unit @@ -251,23 +253,25 @@ class BlockchainImpl( } def saveBestBlock(bestBlock: Option[BigInt]): Unit = { - bestBlock.fold(appStateStorage.putBestBlockNumber(getBestBlockNumber()))(best => appStateStorage.putBestBlockNumber(best)) + bestBlock.fold(appStateStorage.putBestBlockNumber(getBestBlockNumber()).commit())(best => appStateStorage.putBestBlockNumber(best).commit()) } def save(block: Block, receipts: Seq[Receipt], totalDifficulty: BigInt, saveAsBestBlock: Boolean): Unit = { - save(block) - save(block.header.hash, receipts) - save(block.header.hash, totalDifficulty) + storeBlock(block) + .and(storeReceipts(block.header.hash, receipts)) + .and(storeTotalDifficulty(block.header.hash, totalDifficulty)) + .commit() + + // not transactional part stateStorage.onBlockSave(block.header.number, appStateStorage.getBestBlockNumber())(saveBestBlock) if (saveAsBestBlock) { saveBestKnownBlock(block.header.number) } } - override def save(blockHeader: BlockHeader): Unit = { + override def storeBlockHeader(blockHeader: BlockHeader): DataSourceBatchUpdate = { val hash = blockHeader.hash - blockHeadersStorage.put(hash, blockHeader) - saveBlockNumberMapping(blockHeader.number, hash) + blockHeadersStorage.put(hash, blockHeader).and(saveBlockNumberMapping(blockHeader.number, hash)) } override def getMptNodeByHash(hash: ByteString): Option[MptNode] = @@ -275,20 +279,22 @@ class BlockchainImpl( override def getTransactionLocation(txHash: ByteString): Option[TransactionLocation] = transactionMappingStorage.get(txHash) - override def save(blockHash: ByteString, blockBody: BlockBody): Unit = { - blockBodiesStorage.put(blockHash, blockBody) - saveTxsLocations(blockHash, blockBody) + override def storeBlockBody(blockHash: ByteString, blockBody: BlockBody): DataSourceBatchUpdate = { + blockBodiesStorage.put(blockHash, blockBody).and(saveTxsLocations(blockHash, blockBody)) } - override def save(blockHash: ByteString, receipts: Seq[Receipt]): Unit = receiptStorage.put(blockHash, receipts) + override def storeReceipts(blockHash: ByteString, receipts: Seq[Receipt]): DataSourceBatchUpdate = + receiptStorage.put(blockHash, receipts) - override def save(hash: ByteString, evmCode: ByteString): Unit = evmCodeStorage.put(hash, evmCode) + override def storeEvmCode(hash: ByteString, evmCode: ByteString): DataSourceBatchUpdate = + evmCodeStorage.put(hash, evmCode) override def saveBestKnownBlock(number: BigInt): Unit = { bestKnownBlock.set(number) } - def save(blockhash: ByteString, td: BigInt): Unit = totalDifficultyStorage.put(blockhash, td) + def storeTotalDifficulty(blockhash: ByteString, td: BigInt): DataSourceBatchUpdate = + totalDifficultyStorage.put(blockhash, td) def saveNode(nodeHash: NodeHash, nodeEncoded: NodeEncoded, blockNumber: BigInt): Unit = { stateStorage.saveNode(nodeHash, nodeEncoded, blockNumber) @@ -297,10 +303,10 @@ class BlockchainImpl( override protected def getHashByBlockNumber(number: BigInt): Option[ByteString] = blockNumberMappingStorage.get(number) - private def saveBlockNumberMapping(number: BigInt, hash: ByteString): Unit = + private def saveBlockNumberMapping(number: BigInt, hash: ByteString): DataSourceBatchUpdate = blockNumberMappingStorage.put(number, hash) - private def removeBlockNumberMapping(number: BigInt): Unit = { + private def removeBlockNumberMapping(number: BigInt): DataSourceBatchUpdate = { blockNumberMappingStorage.remove(number) } @@ -309,26 +315,39 @@ class BlockchainImpl( val maybeTxList = getBlockBodyByHash(blockHash).map(_.transactionList) val bestSavedBlock = getBestBlockNumber() + val blockNumberMappingUpdates = { + maybeBlockHeader.fold(blockNumberMappingStorage.emptyBatchUpdate)( h => + if (getHashByBlockNumber(h.number).contains(blockHash)) + removeBlockNumberMapping(h.number) + else blockNumberMappingStorage.emptyBatchUpdate + ) + } + blockHeadersStorage.remove(blockHash) - blockBodiesStorage.remove(blockHash) - totalDifficultyStorage.remove(blockHash) - receiptStorage.remove(blockHash) - maybeTxList.foreach(removeTxsLocations) - maybeBlockHeader.foreach{ h => + .and(blockBodiesStorage.remove(blockHash)) + .and(totalDifficultyStorage.remove(blockHash)) + .and(receiptStorage.remove(blockHash)) + .and(maybeTxList.fold(transactionMappingStorage.emptyBatchUpdate)(removeTxsLocations)) + .and(blockNumberMappingUpdates) + .commit() + + // not transactional part + maybeBlockHeader.foreach { h => if (withState) stateStorage.onBlockRollback(h.number, bestSavedBlock)(saveBestBlock) - - if (getHashByBlockNumber(h.number).contains(blockHash)) - removeBlockNumberMapping(h.number) } } - private def saveTxsLocations(blockHash: ByteString, blockBody: BlockBody): Unit = - blockBody.transactionList.zipWithIndex.foreach{ case (tx, index) => - transactionMappingStorage.put(tx.hash, TransactionLocation(blockHash, index)) } + private def saveTxsLocations(blockHash: ByteString, blockBody: BlockBody): DataSourceBatchUpdate = + blockBody.transactionList.zipWithIndex.foldLeft(transactionMappingStorage.emptyBatchUpdate) { + case (updates, (tx, index)) => + updates.and(transactionMappingStorage.put(tx.hash, TransactionLocation(blockHash, index))) + } - private def removeTxsLocations(stxs: Seq[SignedTransaction]): Unit = { - stxs.map(_.hash).foreach{ transactionMappingStorage.remove } + private def removeTxsLocations(stxs: Seq[SignedTransaction]): DataSourceBatchUpdate = { + stxs.map(_.hash).foldLeft(transactionMappingStorage.emptyBatchUpdate) { + case (updates, hash) => updates.and(transactionMappingStorage.remove(hash)) + } } override type S = InMemoryWorldStateProxyStorage @@ -368,7 +387,7 @@ class BlockchainImpl( //FIXME EC-495 this method should not be need when best block is handled properly during rollback def persistCachedNodes(): Unit = { if (stateStorage.forcePersist(RollBackFlush)){ - appStateStorage.putBestBlockNumber(getBestBlockNumber()) + appStateStorage.putBestBlockNumber(getBestBlockNumber()).commit() } } } diff --git a/src/main/scala/io/iohk/ethereum/ledger/InMemoryWorldStateProxy.scala b/src/main/scala/io/iohk/ethereum/ledger/InMemoryWorldStateProxy.scala index 14d7b85a7c..a5d4600a50 100644 --- a/src/main/scala/io/iohk/ethereum/ledger/InMemoryWorldStateProxy.scala +++ b/src/main/scala/io/iohk/ethereum/ledger/InMemoryWorldStateProxy.scala @@ -54,10 +54,10 @@ object InMemoryWorldStateProxy { worldState.accountCodes.foldLeft(worldState) { case (updatedWorldState, (address, code)) => val codeHash = kec256(code) + updatedWorldState.evmCodeStorage.put(codeHash, code).commit() updatedWorldState.copyWith( accountsStateTrie = updatedWorldState.accountsStateTrie + (address -> updatedWorldState.getGuaranteedAccount(address).copy(codeHash = codeHash)), - evmCodeStorage = updatedWorldState.evmCodeStorage + (codeHash -> code), accountCodes = Map.empty ) } diff --git a/src/main/scala/io/iohk/ethereum/network/KnownNodesManager.scala b/src/main/scala/io/iohk/ethereum/network/KnownNodesManager.scala index 1e774666b2..1f785fcf22 100644 --- a/src/main/scala/io/iohk/ethereum/network/KnownNodesManager.scala +++ b/src/main/scala/io/iohk/ethereum/network/KnownNodesManager.scala @@ -59,7 +59,7 @@ class KnownNodesManager( if (toAdd.nonEmpty || toRemove.nonEmpty) { knownNodesStorage.updateKnownNodes( toAdd = toAdd, - toRemove = toRemove) + toRemove = toRemove).commit() toAdd = Set.empty toRemove = Set.empty } diff --git a/src/main/scala/io/iohk/ethereum/nodebuilder/NodeBuilder.scala b/src/main/scala/io/iohk/ethereum/nodebuilder/NodeBuilder.scala index 749bba411d..d899d11127 100644 --- a/src/main/scala/io/iohk/ethereum/nodebuilder/NodeBuilder.scala +++ b/src/main/scala/io/iohk/ethereum/nodebuilder/NodeBuilder.scala @@ -76,9 +76,9 @@ trait PruningConfigBuilder extends PruningModeComponent { } trait StorageBuilder { - lazy val storagesInstance: DataSourcesComponent with StoragesComponent with PruningModeComponent = + lazy val storagesInstance: DataSourceComponent with StoragesComponent with PruningModeComponent = Config.Db.dataSource match { - case "rocksdb" => new SharedRocksDbDataSources with PruningConfigBuilder with Storages.DefaultStorages + case "rocksdb" => new RocksDbDataSourceComponent with PruningConfigBuilder with Storages.DefaultStorages } } diff --git a/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala b/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala index d0ee2e146c..2a3b736802 100644 --- a/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala +++ b/src/main/scala/io/iohk/ethereum/nodebuilder/StdNode.scala @@ -89,7 +89,7 @@ abstract class BaseNode extends Node { tryAndLogFailure(() => consensus.stopProtocol()) tryAndLogFailure(() => Await.ready(system.terminate, shutdownTimeoutDuration)) - tryAndLogFailure(() => storagesInstance.dataSources.closeAll()) + tryAndLogFailure(() => storagesInstance.dataSource.close()) if (jsonRpcConfig.ipcServerConfig.enabled) { tryAndLogFailure(() => jsonRpcIpcServer.close()) } diff --git a/src/snappy/scala/io/iohk/ethereum/snappy/Prerequisites.scala b/src/snappy/scala/io/iohk/ethereum/snappy/Prerequisites.scala index 7aa4c1ff27..ef3439e7af 100644 --- a/src/snappy/scala/io/iohk/ethereum/snappy/Prerequisites.scala +++ b/src/snappy/scala/io/iohk/ethereum/snappy/Prerequisites.scala @@ -5,16 +5,16 @@ import java.util.concurrent.Executors import io.iohk.ethereum.blockchain.data.GenesisDataLoader import io.iohk.ethereum.consensus.StdTestConsensusBuilder import io.iohk.ethereum.db.components.Storages.PruningModeComponent -import io.iohk.ethereum.db.components.{ DataSourcesComponent, SharedRocksDbDataSources, Storages } +import io.iohk.ethereum.db.components.{DataSourceComponent, RocksDbDataSourceComponent, Storages} import io.iohk.ethereum.db.dataSource._ import io.iohk.ethereum.db.storage.Namespaces -import io.iohk.ethereum.db.storage.pruning.{ ArchivePruning, PruningMode } +import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode} import io.iohk.ethereum.domain.BlockchainImpl import io.iohk.ethereum.ledger.Ledger.VMImpl -import io.iohk.ethereum.ledger.{ Ledger, LedgerImpl } +import io.iohk.ethereum.ledger.{Ledger, LedgerImpl} import io.iohk.ethereum.nodebuilder._ -import io.iohk.ethereum.snappy.Config.{ DualDB, SingleDB } -import io.iohk.ethereum.snappy.Prerequisites.{RocksDbStorages, Storages } +import io.iohk.ethereum.snappy.Config.{DualDB, SingleDB} +import io.iohk.ethereum.snappy.Prerequisites.{RocksDbStorages, Storages} import scala.concurrent.ExecutionContext @@ -23,11 +23,11 @@ object Prerequisites { val pruningMode: PruningMode = ArchivePruning } - trait Storages extends DataSourcesComponent with NoPruning with Storages.DefaultStorages { + trait Storages extends DataSourceComponent with NoPruning with Storages.DefaultStorages { val dataSource: DataSource } - trait RocksDbStorages extends SharedRocksDbDataSources with Storages + trait RocksDbStorages extends RocksDbDataSourceComponent with Storages } class Prerequisites(config: Config) { diff --git a/src/snappy/scala/io/iohk/ethereum/snappy/SnappyTest.scala b/src/snappy/scala/io/iohk/ethereum/snappy/SnappyTest.scala index 0dc2687133..2d2039153d 100644 --- a/src/snappy/scala/io/iohk/ethereum/snappy/SnappyTest.scala +++ b/src/snappy/scala/io/iohk/ethereum/snappy/SnappyTest.scala @@ -40,8 +40,11 @@ class SnappyTest extends FreeSpec with Matchers with Logger { case Right(receipts) => if (receipts == expectedReceipts) { - targetBlockchain.foreach(_.save(block)) - targetBlockchain.foreach(_.save(block.header.hash, receipts)) + targetBlockchain.foreach { blockchain => + blockchain.storeBlock(block) + .and(blockchain.storeReceipts(block.header.hash, receipts)) + .commit() + } } else { fail(s"Block $n did not execute correctly.\n$receipts did not equal $expectedReceipts") } diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/BlockchainHostActorSpec.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/BlockchainHostActorSpec.scala index 4f284190cf..95240d3661 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/BlockchainHostActorSpec.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/BlockchainHostActorSpec.scala @@ -34,8 +34,9 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val receipts: Seq[Seq[Receipt]] = Seq(Seq(),Seq()) - blockchain.save(receiptsHashes(0), receipts(0)) - blockchain.save(receiptsHashes(1), receipts(1)) + blockchain.storeReceipts(receiptsHashes(0), receipts(0)) + .and(blockchain.storeReceipts(receiptsHashes(1), receipts(1))) + .commit() //when blockchainHost ! MessageFromPeer(GetReceipts(receiptsHashes), peerId) @@ -52,8 +53,9 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val blockBodies = Seq(baseBlockBody,baseBlockBody) - blockchain.save(blockBodiesHashes(0), blockBodies(0)) - blockchain.save(blockBodiesHashes(1), blockBodies(1)) + blockchain.storeBlockBody(blockBodiesHashes(0), blockBodies(0)) + .and(blockchain.storeBlockBody(blockBodiesHashes(1), blockBodies(1))) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockBodies(blockBodiesHashes), peerId) @@ -67,10 +69,11 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 3) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 4) - blockchain.save(firstHeader) - blockchain.save(secondHeader) - blockchain.save(baseBlockHeader.copy(number = 5)) - blockchain.save(baseBlockHeader.copy(number = 6)) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(secondHeader)) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 5))) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 6))) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockHeaders(Left(3), 2, 0, reverse = false), peerId) @@ -84,8 +87,9 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 3) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 4) - blockchain.save(firstHeader) - blockchain.save(secondHeader) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(secondHeader)) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockHeaders(Left(3), 3, 0, reverse = false), peerId) @@ -99,9 +103,10 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 3) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 2) - blockchain.save(firstHeader) - blockchain.save(secondHeader) - blockchain.save(baseBlockHeader.copy(number = 1)) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(secondHeader)) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 1))) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockHeaders(Left(3), 2, 0, reverse = true), peerId) @@ -115,10 +120,11 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 3) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 4) - blockchain.save(firstHeader) - blockchain.save(secondHeader) - blockchain.save(baseBlockHeader.copy(number = 5)) - blockchain.save(baseBlockHeader.copy(number = 6)) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(secondHeader)) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 5))) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 6))) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockHeaders(Right(firstHeader.hash), 2, 0, reverse = false), peerId) @@ -132,11 +138,12 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 3) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 5) - blockchain.save(firstHeader) - blockchain.save(baseBlockHeader.copy(number = 4)) - blockchain.save(secondHeader) - blockchain.save(baseBlockHeader.copy(number = 6)) - blockchain.save(baseBlockHeader.copy(number = 7)) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 4))) + .and(blockchain.storeBlockHeader(secondHeader)) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 6))) + .and(blockchain.storeBlockHeader(baseBlockHeader.copy(number = 7))) + .commit() //when blockchainHost ! MessageFromPeer( @@ -151,8 +158,9 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 3) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 1) - blockchain.save(firstHeader) - blockchain.save(secondHeader) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(secondHeader)) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockHeaders(Right(firstHeader.hash), 2, 1, reverse = true), peerId) @@ -166,8 +174,9 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 3) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 1) - blockchain.save(firstHeader) - blockchain.save(secondHeader) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(secondHeader)) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockHeaders(Right(firstHeader.hash), 3, 1, reverse = true), peerId) @@ -181,8 +190,9 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val firstHeader: BlockHeader = baseBlockHeader.copy(number = 4) val secondHeader: BlockHeader = baseBlockHeader.copy(number = 2) - blockchain.save(firstHeader) - blockchain.save(secondHeader) + blockchain.storeBlockHeader(firstHeader) + .and(blockchain.storeBlockHeader(secondHeader)) + .commit() //when blockchainHost ! MessageFromPeer(GetBlockHeaders(Right(firstHeader.hash), 4, 1, reverse = true), peerId) @@ -197,7 +207,7 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { val fakeEvmCode = ByteString(Hex.decode("ffddaaffddaaffddaaffddaaffddaa")) val evmCodeHash: ByteString = ByteString(crypto.kec256(fakeEvmCode.toArray[Byte])) - blockchain.save(evmCodeHash, fakeEvmCode) + blockchain.storeEvmCode(evmCodeHash, fakeEvmCode).commit() //when blockchainHost ! MessageFromPeer(GetNodeData(Seq(evmCodeHash)), peerId) @@ -224,7 +234,7 @@ class BlockchainHostActorSpec extends FlatSpec with Matchers { trait TestSetup extends EphemBlockchainTestSetup { override implicit lazy val system = ActorSystem("BlockchainHostActor_System") - blockchain.save(Fixtures.Blocks.Genesis.header) + blockchain.storeBlockHeader(Fixtures.Blocks.Genesis.header).commit() val peerConf = new PeerConfiguration { override val fastSyncHostConfiguration: FastSyncHostConfiguration = new FastSyncHostConfiguration { diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala index be8d6ada7a..9e53859ba9 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/EphemBlockchainTestSetup.scala @@ -1,6 +1,6 @@ package io.iohk.ethereum.blockchain.sync -import io.iohk.ethereum.db.components.{SharedEphemDataSources, Storages} +import io.iohk.ethereum.db.components.{EphemDataSourceComponent, Storages} import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode} import io.iohk.ethereum.ledger.Ledger.VMImpl import io.iohk.ethereum.nodebuilder.PruningConfigBuilder @@ -13,6 +13,6 @@ trait EphemBlockchainTestSetup extends ScenarioSetup { //+ cake overrides override lazy val vm: VMImpl = new VMImpl - override lazy val storagesInstance = new SharedEphemDataSources with LocalPruningConfigBuilder with Storages.DefaultStorages + override lazy val storagesInstance = new EphemDataSourceComponent with LocalPruningConfigBuilder with Storages.DefaultStorages //- cake overrides } diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncControllerSpec.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncControllerSpec.scala index d348c2309d..bc2b4ae8d6 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncControllerSpec.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/SyncControllerSpec.scala @@ -607,7 +607,7 @@ class SyncControllerSpec extends FlatSpec with Matchers with BeforeAndAfter with val EmptyTrieRootHash: ByteString = Account.EmptyStorageRootHash val baseBlockHeader = Fixtures.Blocks.Genesis.header - blockchain.save(baseBlockHeader.parentHash, BigInt(0)) + blockchain.storeTotalDifficulty(baseBlockHeader.parentHash, BigInt(0)).commit() val startDelayMillis = 200 diff --git a/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/OldRegularSyncSpec.scala b/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/OldRegularSyncSpec.scala index a8f1176962..8f67da21c4 100644 --- a/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/OldRegularSyncSpec.scala +++ b/src/test/scala/io/iohk/ethereum/blockchain/sync/regular/OldRegularSyncSpec.scala @@ -204,7 +204,7 @@ class OldRegularSyncSpec extends WordSpec with Matchers with MockFactory with Ev "blacklist peer sending ancient block hashes" in new TestSetup { startSyncing() val blockHash: BlockHash = randomBlockHash() - storagesInstance.storages.appStateStorage.putBestBlockNumber(blockHash.number + syncConfig.maxNewBlockHashAge + 1) + storagesInstance.storages.appStateStorage.putBestBlockNumber(blockHash.number + syncConfig.maxNewBlockHashAge + 1).commit() sendBlockHeaders(Seq.empty) sendNewBlockHashMsg(Seq(blockHash)) @@ -573,7 +573,7 @@ class OldRegularSyncSpec extends WordSpec with Matchers with MockFactory with Ev Future.successful(a) } - storagesInstance.storages.appStateStorage.putBestBlockNumber(0) + storagesInstance.storages.appStateStorage.putBestBlockNumber(0).commit() val etcPeerManager = TestProbe() val peerEventBus = TestProbe() diff --git a/src/test/scala/io/iohk/ethereum/consensus/validators/BlockHeaderValidatorSpec.scala b/src/test/scala/io/iohk/ethereum/consensus/validators/BlockHeaderValidatorSpec.scala index 959f52cc2a..ecfbaa3b17 100644 --- a/src/test/scala/io/iohk/ethereum/consensus/validators/BlockHeaderValidatorSpec.scala +++ b/src/test/scala/io/iohk/ethereum/consensus/validators/BlockHeaderValidatorSpec.scala @@ -160,8 +160,9 @@ class BlockHeaderValidatorSpec } it should "validate correctly a block whose parent is in storage" in new EphemBlockchainTestSetup { - blockchain.save(validParentBlockHeader) - blockchain.save(validParentBlockHeader.hash, validParentBlockBody) + blockchain.storeBlockHeader(validParentBlockHeader) + .and(blockchain.storeBlockBody(validParentBlockHeader.hash, validParentBlockBody)) + .commit() blockHeaderValidator.validate(validBlockHeader, blockchain.getBlockHeaderByHash _) match { case Right(_) => succeed case _ => fail diff --git a/src/test/scala/io/iohk/ethereum/consensus/validators/OmmersValidatorSpec.scala b/src/test/scala/io/iohk/ethereum/consensus/validators/OmmersValidatorSpec.scala index c855499008..2548fc3a42 100644 --- a/src/test/scala/io/iohk/ethereum/consensus/validators/OmmersValidatorSpec.scala +++ b/src/test/scala/io/iohk/ethereum/consensus/validators/OmmersValidatorSpec.scala @@ -372,14 +372,15 @@ class OmmersValidatorSpec extends FlatSpec with Matchers with ScalaCheckProperty val ommersBlockParentHash: ByteString = block96.header.hash - blockchain.save(block89) - blockchain.save(block90) - blockchain.save(block91) - blockchain.save(block92) - blockchain.save(block93) - blockchain.save(block94) - blockchain.save(block95) - blockchain.save(block96) + blockchain.storeBlock(block89) + .and(blockchain.storeBlock(block90)) + .and(blockchain.storeBlock(block91)) + .and(blockchain.storeBlock(block92)) + .and(blockchain.storeBlock(block93)) + .and(blockchain.storeBlock(block94)) + .and(blockchain.storeBlock(block95)) + .and(blockchain.storeBlock(block96)) + .commit() } } diff --git a/src/test/scala/io/iohk/ethereum/db/dataSource/DataSourceTestBehavior.scala b/src/test/scala/io/iohk/ethereum/db/dataSource/DataSourceTestBehavior.scala index 076cfc2f95..3fc40e09b0 100644 --- a/src/test/scala/io/iohk/ethereum/db/dataSource/DataSourceTestBehavior.scala +++ b/src/test/scala/io/iohk/ethereum/db/dataSource/DataSourceTestBehavior.scala @@ -3,6 +3,7 @@ package io.iohk.ethereum.db.dataSource import java.io.File import java.nio.file.Files import io.iohk.ethereum.ObjectGenerators +import io.iohk.ethereum.db.dataSource.DataSource.{Key, Namespace, Value} import org.scalatest.FlatSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks @@ -13,6 +14,13 @@ trait DataSourceTestBehavior extends ScalaCheckPropertyChecks with ObjectGenerat val KeySizeWithoutPrefix: Int = 32 val OtherNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('r'.toByte) + def prepareUpdate( + namespace: Namespace = OtherNamespace, + toRemove: Seq[Key] = Nil, + toUpsert: Seq[(Key, Value)] = Nil + ): Seq[DataSourceUpdate] = + Seq(DataSourceUpdate(namespace, toRemove, toUpsert)) + def withDir(testCode: String => Any): Unit = { val path = Files.createTempDirectory("testdb").getFileName.toString try { @@ -29,7 +37,7 @@ trait DataSourceTestBehavior extends ScalaCheckPropertyChecks with ObjectGenerat val someByteString = byteStringOfLengthNGen(KeySizeWithoutPrefix).sample.get withDir { path => val dataSource = createDataSource(path) - dataSource.update(OtherNamespace, Seq(), Seq(someByteString -> someByteString)) + dataSource.update(prepareUpdate(toUpsert = Seq(someByteString -> someByteString))) dataSource.get(OtherNamespace, someByteString) match { case Some(b) if b == someByteString => succeed @@ -46,12 +54,12 @@ trait DataSourceTestBehavior extends ScalaCheckPropertyChecks with ObjectGenerat withDir { path => val dataSource = createDataSource(path) - dataSource.update(OtherNamespace, Seq(), Seq(key1 -> key1, key2 -> key2)) + dataSource.update(prepareUpdate(toUpsert = Seq(key1 -> key1, key2 -> key2))) assert(dataSource.get(OtherNamespace, key1).isDefined) assert(dataSource.get(OtherNamespace, key2).isDefined) - dataSource.update(OtherNamespace, Seq(key1), Seq()) + dataSource.update(prepareUpdate(toRemove = Seq(key1))) assert(dataSource.get(OtherNamespace, key1).isEmpty) assert(dataSource.get(OtherNamespace, key2).isDefined) @@ -65,13 +73,13 @@ trait DataSourceTestBehavior extends ScalaCheckPropertyChecks with ObjectGenerat withDir { path => val dataSource = createDataSource(path) - dataSource.update(OtherNamespace, Seq(), Seq(someByteString -> someByteString)) + dataSource.update(prepareUpdate(toUpsert = Seq(someByteString -> someByteString))) assert(dataSource.get(OtherNamespace, someByteString).isDefined) - val newDataSource = dataSource.clear + dataSource.clear() - assert(newDataSource.get(OtherNamespace, someByteString).isEmpty) + assert(dataSource.get(OtherNamespace, someByteString).isEmpty) dataSource.destroy() } @@ -86,14 +94,14 @@ trait DataSourceTestBehavior extends ScalaCheckPropertyChecks with ObjectGenerat val dataSource = createDataSource(path) //Insertion - dataSource.update(OtherNamespace, Seq(), Seq(someByteString -> someValue1)) - dataSource.update(OtherNamespace2, Seq(), Seq(someByteString -> someValue2)) + dataSource.update(prepareUpdate(namespace = OtherNamespace, toUpsert = Seq(someByteString -> someValue1))) + dataSource.update(prepareUpdate(namespace = OtherNamespace2, toUpsert = Seq(someByteString -> someValue2))) assert(dataSource.get(OtherNamespace, someByteString).contains(someValue1)) assert(dataSource.get(OtherNamespace2, someByteString).contains(someValue2)) //Removal - dataSource.update(OtherNamespace2, Seq(someByteString), Nil) + dataSource.update(prepareUpdate(namespace = OtherNamespace2, toRemove = Seq(someByteString))) assert(dataSource.get(OtherNamespace, someByteString).contains(someValue1)) assert(dataSource.get(OtherNamespace2, someByteString).isEmpty) diff --git a/src/test/scala/io/iohk/ethereum/db/dataSource/EphemDataSourceSuite.scala b/src/test/scala/io/iohk/ethereum/db/dataSource/EphemDataSourceSuite.scala index 7307ef1cf9..a5551bad29 100644 --- a/src/test/scala/io/iohk/ethereum/db/dataSource/EphemDataSourceSuite.scala +++ b/src/test/scala/io/iohk/ethereum/db/dataSource/EphemDataSourceSuite.scala @@ -11,22 +11,24 @@ class EphemDataSourceSuite extends FunSuite with ScalaCheckPropertyChecks with O val KeySize: Int = 32 val KeyNumberLimit: Int = 40 val OtherNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('e'.toByte) - def putMultiple(dataSource: DataSource, toInsert: Seq[(ByteString, ByteString)]): DataSource = { - toInsert.foldLeft(dataSource) { case (recDB, keyValuePair) => - recDB.update(OtherNamespace, Seq(), Seq(keyValuePair)) + + def putMultiple(dataSource: DataSource, toInsert: Seq[(ByteString, ByteString)]): Unit = { + toInsert.foreach { keyValuePair => + dataSource.update(Seq(DataSourceUpdate(OtherNamespace, Seq(), Seq(keyValuePair)))) } } - def removeMultiple(dataSource: DataSource, toDelete: Seq[ByteString]): DataSource = { - toDelete.foldLeft(dataSource) { case (recDB, key) => - recDB.update(OtherNamespace, Seq(key), Seq()) + def removeMultiple(dataSource: DataSource, toDelete: Seq[ByteString]): Unit = { + toDelete.foreach { key => + dataSource.update(Seq(DataSourceUpdate(OtherNamespace, Seq(key), Seq()))) } } test("EphemDataSource insert") { forAll(seqByteStringOfNItemsGen(KeySize)) { unFilteredKeyList: Seq[ByteString] => val keyList = unFilteredKeyList.filter(_.length == KeySize) - val db = putMultiple(dataSource = EphemDataSource(), toInsert = keyList.zip(keyList)) + val db = EphemDataSource() + putMultiple(dataSource = db, toInsert = keyList.zip(keyList)) keyList.foreach { key => val obtained = db.get(OtherNamespace, key) assert(obtained.isDefined) @@ -39,24 +41,31 @@ class EphemDataSourceSuite extends FunSuite with ScalaCheckPropertyChecks with O forAll(seqByteStringOfNItemsGen(KeySize)) { keyList: Seq[ByteString] => val (keysToDelete, keyValueLeft) = keyList.splitAt(Gen.choose(0, keyList.size).sample.get) - val dbAfterInsert = putMultiple(dataSource = EphemDataSource(), toInsert = keyList.zip(keyList)) - val db = removeMultiple(dataSource = dbAfterInsert, toDelete = keysToDelete) + val db = EphemDataSource() + putMultiple(dataSource = db, toInsert = keyList.zip(keyList)) + removeMultiple(dataSource = db, toDelete = keysToDelete) + keyValueLeft.foreach { key => val obtained = db.get(OtherNamespace, key) assert(obtained.isDefined) assert(obtained.get sameElements key) } - keysToDelete.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) } + keysToDelete.foreach { key => + assert(db.get(OtherNamespace, key).isEmpty) + } } } test("EphemDataSource clear") { forAll(seqByteStringOfNItemsGen(KeySize)) { keyList: Seq[ByteString] => val db = EphemDataSource() - .update(OtherNamespace, toRemove = Seq(), toUpsert = keyList.zip(keyList)) - .clear - keyList.foreach { key => assert(db.get(OtherNamespace, key).isEmpty) } + putMultiple(db, keyList.zip(keyList)) + db.clear() + + keyList.foreach { key => + assert(db.get(OtherNamespace, key).isEmpty) + } } } } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/BlockBodiesStorageSpec.scala b/src/test/scala/io/iohk/ethereum/db/storage/BlockBodiesStorageSpec.scala index 0622c4edfb..a8a1309657 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/BlockBodiesStorageSpec.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/BlockBodiesStorageSpec.scala @@ -2,6 +2,7 @@ package io.iohk.ethereum.db.storage import io.iohk.ethereum.ObjectGenerators import io.iohk.ethereum.db.dataSource.EphemDataSource +import io.iohk.ethereum.network.p2p.messages.CommonMessages import io.iohk.ethereum.network.p2p.messages.CommonMessages.NewBlock import io.iohk.ethereum.nodebuilder.SecureRandomBuilder import org.bouncycastle.util.encoders.Hex @@ -10,7 +11,7 @@ import org.scalatest.WordSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks class BlockBodiesStorageSpec - extends WordSpec + extends WordSpec with ScalaCheckPropertyChecks with ObjectGenerators with SecureRandomBuilder { @@ -21,42 +22,45 @@ class BlockBodiesStorageSpec "insert block body properly" in { forAll(Gen.listOfN(32, ObjectGenerators.newBlockGen(secureRandom, chainId))) { newBlocks => - val initialStorage = new BlockBodiesStorage(EphemDataSource()) val blocks = newBlocks.distinct - val totalStorage = newBlocks.foldLeft(initialStorage) { case (storage, NewBlock(block, _)) => - storage.put(block.header.hash, block.body) - } + val totalStorage = insertBlockBodiesMapping(newBlocks) - blocks.foreach { case NewBlock(block, _) => - assert(totalStorage.get(block.header.hash).contains(block.body)) + blocks.foreach { + case NewBlock(block, _) => assert(totalStorage.get(block.header.hash).contains(block.body)) } } } "delete block body properly" in { forAll(Gen.listOfN(32, ObjectGenerators.newBlockGen(secureRandom, chainId))) { newBlocks => - val initialStorage = new BlockBodiesStorage(EphemDataSource()) val blocks = newBlocks.distinct - val totalStorage = newBlocks.foldLeft(initialStorage) { case (storage, NewBlock(block, _)) => - storage.put(block.header.hash, block.body) - } - - // Mapping of block bodies is inserted - blocks.foreach { case NewBlock(block, _) => assert(totalStorage.get(block.header.hash).contains(block.body)) } + val storage = insertBlockBodiesMapping(newBlocks) // Mapping of block bodies is deleted val (toDelete, toLeave) = blocks.splitAt(Gen.choose(0, blocks.size).sample.get) - val storageAfterDelete = toDelete.foldLeft(totalStorage) { case (storage, NewBlock(block, _)) => - storage.remove(block.header.hash) + val batchUpdates = toDelete.foldLeft(storage.emptyBatchUpdate) { + case (updates, NewBlock(block, _)) => updates.and(storage.remove(block.header.hash)) } - toLeave.foreach { case NewBlock(block, _) => - assert(storageAfterDelete.get(block.header.hash).contains(block.body)) + batchUpdates.commit() + + toLeave.foreach { + case NewBlock(block, _) => assert(storage.get(block.header.hash).contains(block.body)) } - toDelete.foreach { case NewBlock(block, _) => assert(storageAfterDelete.get(block.header.hash).isEmpty) } + toDelete.foreach { case NewBlock(block, _) => assert(storage.get(block.header.hash).isEmpty) } + } + } + def insertBlockBodiesMapping(newBlocks: Seq[CommonMessages.NewBlock]): BlockBodiesStorage = { + val storage = new BlockBodiesStorage(EphemDataSource()) + + val batchUpdates = newBlocks.foldLeft(storage.emptyBatchUpdate) { + case (updates, NewBlock(block, _)) => updates.and(storage.put(block.header.hash, block.body)) } + + batchUpdates.commit() + storage } } } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/BlockHeadersStorageSpec.scala b/src/test/scala/io/iohk/ethereum/db/storage/BlockHeadersStorageSpec.scala index f9c4cfa0e2..588d75a963 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/BlockHeadersStorageSpec.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/BlockHeadersStorageSpec.scala @@ -12,37 +12,40 @@ class BlockHeadersStorageSpec extends WordSpec with ScalaCheckPropertyChecks wit "BlockHeadersStorage" should { "insert block header properly" in { - forAll(Gen.listOfN(32, ObjectGenerators.blockHeaderGen)) { blockHeaders => - val initialStorage = new BlockHeadersStorage(EphemDataSource()) + forAll(Gen.listOfN(32, ObjectGenerators.blockHeaderGen)){ blockHeaders => + val storage = new BlockHeadersStorage(EphemDataSource()) val headers = blockHeaders.distinct - val totalStorage = blockHeaders.foldLeft(initialStorage) { case (storage, blockHeader) => - storage.put(blockHeader.hash, blockHeader) + val storagesUpdates = blockHeaders.foldLeft(storage.emptyBatchUpdate) { + case (updates, blockHeader) => updates.and(storage.put(blockHeader.hash, blockHeader)) } + storagesUpdates.commit() - checkIfIsInStorage(headers, totalStorage) + checkIfIsInStorage(headers, storage) } } "delete block header properly" in { - forAll(Gen.listOfN(32, ObjectGenerators.blockHeaderGen)) { blockHeaders => - val initialStorage = new BlockHeadersStorage(EphemDataSource()) + forAll(Gen.listOfN(32, ObjectGenerators.blockHeaderGen)){ blockHeaders => + val storage = new BlockHeadersStorage(EphemDataSource()) val headers = blockHeaders.distinct - val totalStorage = blockHeaders.foldLeft(initialStorage) { case (storage, blockHeader) => - storage.put(blockHeader.hash, blockHeader) + val storageInsertions = blockHeaders.foldLeft(storage.emptyBatchUpdate) { + case (updates, blockHeader) => updates.and(storage.put(blockHeader.hash, blockHeader)) } + storageInsertions.commit() // Mapping of block headers is inserted - checkIfIsInStorage(headers, totalStorage) + checkIfIsInStorage(headers, storage) // Mapping of block headers is deleted val (toDelete, toLeave) = headers.splitAt(Gen.choose(0, headers.size).sample.get) - val storageAfterDelete = toDelete.foldLeft(totalStorage) { case (storage, blockHeader) => - storage.remove(blockHeader.hash) + val storageDeletions = toDelete.foldLeft(storage.emptyBatchUpdate) { + case (updates, blockHeader) => updates.and(storage.remove(blockHeader.hash)) } + storageDeletions.commit() - checkIfIsInStorage(toLeave, storageAfterDelete) - toDelete.foreach(header => assert(storageAfterDelete.get(header.hash).isEmpty)) + checkIfIsInStorage(toLeave, storage) + toDelete.foreach(header => assert(storage.get(header.hash).isEmpty)) } } } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/CodeStorageSuite.scala b/src/test/scala/io/iohk/ethereum/db/storage/CodeStorageSuite.scala index 366cf90231..14b63c4151 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/CodeStorageSuite.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/CodeStorageSuite.scala @@ -14,13 +14,16 @@ class CodeStorageSuite extends FunSuite with ScalaCheckPropertyChecks with Objec forAll(Gen.listOf(byteStringOfLengthNGen(32))) { unfilteredCodeHashes => val codeHashes = unfilteredCodeHashes.distinct val codes = Gen.listOfN(codeHashes.length, randomSizeByteArrayGen(0, LimitCodeSize)).sample.get.map(ByteString(_)) - val initialCodeStorage = new EvmCodeStorage(EphemDataSource()) - val codeStorage = codeHashes.zip(codes).foldLeft(initialCodeStorage) { case (recCodeStorage, (codeHash, code)) => - recCodeStorage.put(codeHash, code) + val storage = new EvmCodeStorage(EphemDataSource()) + val batchUpdates = codeHashes.zip(codes).foldLeft(storage.emptyBatchUpdate) { + case (updates, (codeHash, code)) => + updates.and(storage.put(codeHash, code)) } + batchUpdates.commit() - codeHashes.zip(codes).foreach { case (codeHash, code) => - assert(codeStorage.get(codeHash).contains(code)) + codeHashes.zip(codes).foreach { + case (codeHash, code) => + assert(storage.get(codeHash).contains(code)) } } } @@ -31,21 +34,25 @@ class CodeStorageSuite extends FunSuite with ScalaCheckPropertyChecks with Objec val codes = Gen.listOfN(codeHashes.length, randomSizeByteArrayGen(0, LimitCodeSize)).sample.get.map(ByteString(_)) //EVM codes are inserted - val initialCodeStorage = new EvmCodeStorage(EphemDataSource()) - val codeStorage = codeHashes.zip(codes).foldLeft(initialCodeStorage) { case (recCodeStorage, (codeHash, code)) => - recCodeStorage.put(codeHash, code) + val storage = new EvmCodeStorage(EphemDataSource()) + val storageInsertions = codeHashes.zip(codes).foldLeft(storage.emptyBatchUpdate) { + case (updates, (codeHash, code)) => + updates.and(storage.put(codeHash, code)) } + storageInsertions.commit() //EVM codes are deleted val (toDelete, toLeave) = codeHashes .zip(codes) .splitAt(Gen.choose(0, codeHashes.size).sample.get) - val codeStorageAfterDelete = toDelete.foldLeft(codeStorage) { case (recCodeStorage, (codeHash, _)) => - recCodeStorage.remove(codeHash) + val storageDeletions = toDelete.foldLeft(storage.emptyBatchUpdate) { + case (updates, (codeHash, _)) => + updates.and(storage.remove(codeHash)) } + storageDeletions.commit() - toLeave.foreach { case (codeHash, code) => assert(codeStorageAfterDelete.get(codeHash).contains(code)) } - toDelete.foreach { case (codeHash, _) => assert(codeStorageAfterDelete.get(codeHash).isEmpty) } + toLeave.foreach { case (codeHash, code) => assert(storage.get(codeHash).contains(code)) } + toDelete.foreach { case (codeHash, _) => assert(storage.get(codeHash).isEmpty) } } } } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/KeyValueStorageSuite.scala b/src/test/scala/io/iohk/ethereum/db/storage/KeyValueStorageSuite.scala index c385ed9757..c4aa2b7d1b 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/KeyValueStorageSuite.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/KeyValueStorageSuite.scala @@ -1,7 +1,7 @@ package io.iohk.ethereum.db.storage import io.iohk.ethereum.ObjectGenerators -import io.iohk.ethereum.db.dataSource.{DataSource, EphemDataSource} +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdate, EphemDataSource} import io.iohk.ethereum.rlp.RLPImplicits._ import io.iohk.ethereum.rlp.{decode => rlpDecode, encode => rlpEncode} import org.scalacheck.Gen @@ -33,13 +33,18 @@ class KeyValueStorageSuite extends FunSuite with ScalaCheckPropertyChecks with O val initialIntStorage = new IntStorage(EphemDataSource()) - test("Get ints from KeyValueStorage") { - forAll(Gen.listOf(intGen), Gen.listOf(intGen)) { (intsInStorage, unfilteredIntsNotInStorage) => - val intsNotInStorage = unfilteredIntsNotInStorage.distinct diff intsInStorage + val dataGenerator = for { + intsInStorage <- Gen.nonEmptyListOf(intGen) + intsNotInStorage <- Gen.nonEmptyListOf(intGen.suchThat(value => !intsInStorage.contains(value))) + } yield (intsInStorage, intsNotInStorage) - val intsInStorageIndexedSeq = intsInStorage.map { IntStorage.intSerializer(_) } + test("Get ints from KeyValueStorage") { + forAll(dataGenerator) { case (intsInStorage, intsNotInStorage) => + val intsInStorageIndexedSeq = intsInStorage.map{ IntStorage.intSerializer(_) } val initialIntDataSource = EphemDataSource() - .update(IntStorage.intNamespace, Seq(), intsInStorageIndexedSeq.zip(intsInStorageIndexedSeq)) + initialIntDataSource.update( + Seq(DataSourceUpdate(IntStorage.intNamespace, Seq(), intsInStorageIndexedSeq.zip(intsInStorageIndexedSeq))) + ) val keyValueStorage = new IntStorage(initialIntDataSource) intsInStorage.foreach { i => assert(keyValueStorage.get(i).contains(i)) } intsNotInStorage.foreach { i => assert(keyValueStorage.get(i).isEmpty) } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/NodeStorageSuite.scala b/src/test/scala/io/iohk/ethereum/db/storage/NodeStorageSuite.scala index 0468c71d8a..17b310a06e 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/NodeStorageSuite.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/NodeStorageSuite.scala @@ -1,3 +1,4 @@ + package io.iohk.ethereum.db.storage import akka.util.ByteString diff --git a/src/test/scala/io/iohk/ethereum/db/storage/ReceiptStorageSuite.scala b/src/test/scala/io/iohk/ethereum/db/storage/ReceiptStorageSuite.scala index 3399a05d73..ca66ac3384 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/ReceiptStorageSuite.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/ReceiptStorageSuite.scala @@ -15,15 +15,17 @@ class ReceiptStorageSuite extends FunSuite with ScalaCheckPropertyChecks with Ob val receipts = receiptsGen(blockHashes.length).sample.get val blockHashesReceiptsPair = receipts.zip(blockHashes) - val initialReceiptStorage = new ReceiptStorage(EphemDataSource()) - val receiptStorage = blockHashesReceiptsPair.foldLeft(initialReceiptStorage) { - case (recReceiptStorage, (receiptList, blockHash)) => - recReceiptStorage.put(blockHash, receiptList) + val storage = new ReceiptStorage(EphemDataSource()) + val batchUpdates = blockHashesReceiptsPair.foldLeft(storage.emptyBatchUpdate) { + case (updates, (receiptList, blockHash)) => + updates.and(storage.put(blockHash, receiptList)) } + batchUpdates.commit() - blockHashesReceiptsPair.foreach { case (rs, bh) => - val obtainedReceipts: Option[Seq[Receipt]] = receiptStorage.get(bh) - assert(obtainedReceipts.contains(rs)) + blockHashesReceiptsPair.foreach { + case (rs, bh) => + val obtainedReceipts: Option[Seq[Receipt]] = storage.get(bh) + assert(obtainedReceipts.contains(rs)) } } } @@ -35,23 +37,27 @@ class ReceiptStorageSuite extends FunSuite with ScalaCheckPropertyChecks with Ob val blockHashesReceiptsPair = receipts.zip(blockHashes) //Receipts are inserted - val initialReceiptStorage = new ReceiptStorage(EphemDataSource()) - val receiptStorage = blockHashesReceiptsPair.foldLeft(initialReceiptStorage) { - case (recReceiptStorage, (receiptList, blockHash)) => - recReceiptStorage.put(blockHash, receiptList) + val storage = new ReceiptStorage(EphemDataSource()) + val storageInsertions = blockHashesReceiptsPair.foldLeft(storage.emptyBatchUpdate) { + case (updates, (receiptList, blockHash)) => + updates.and(storage.put(blockHash, receiptList)) } + storageInsertions.commit() //Receipts are deleted val (toDelete, toLeave) = blockHashesReceiptsPair.splitAt(Gen.choose(0, blockHashesReceiptsPair.size).sample.get) - val receiptStorageAfterDelete = toDelete.foldLeft(receiptStorage) { case (recReceiptStorage, (_, blockHash)) => - recReceiptStorage.remove(blockHash) + val storageDeletions = toDelete.foldLeft(storage.emptyBatchUpdate) { + case (updates, (_, blockHash)) => + updates.and(storage.remove(blockHash)) } + storageDeletions.commit() - toLeave.foreach { case (rs, bh) => - val obtainedReceipts = receiptStorageAfterDelete.get(bh) - assert(obtainedReceipts.contains(rs)) + toLeave.foreach { + case (rs, bh) => + val obtainedReceipts = storage.get(bh) + assert(obtainedReceipts.contains(rs)) } - toDelete.foreach { case (_, bh) => assert(receiptStorageAfterDelete.get(bh).isEmpty) } + toDelete.foreach { case (_, bh) => assert(storage.get(bh).isEmpty) } } } } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorageSpec.scala b/src/test/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorageSpec.scala index 2a8d35e736..4786486ecb 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorageSpec.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/ReferenceCountNodeStorageSpec.scala @@ -313,5 +313,3 @@ class ReferenceCountNodeStorageSpec extends FlatSpec with Matchers { } } - - diff --git a/src/test/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorageSuite.scala b/src/test/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorageSuite.scala index 55cd1d501a..16f5e16c43 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorageSuite.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/TotalDifficultyStorageSuite.scala @@ -13,13 +13,14 @@ class TotalDifficultyStorageSuite extends FunSuite with ScalaCheckPropertyChecks val tdList = Gen.listOf(bigIntGen).sample.get val blockHashesTdPair = tdList.zip(blockHashes) - val initialTotalDifficultyStorage = new TotalDifficultyStorage(EphemDataSource()) - val totalDifficultyStorage = blockHashesTdPair.foldLeft(initialTotalDifficultyStorage) { - case (recTotalDifficultyStorage, (td, blockHash)) => - recTotalDifficultyStorage.put(blockHash, td) + val storage = new TotalDifficultyStorage(EphemDataSource()) + val batchUpdates = blockHashesTdPair.foldLeft(storage.emptyBatchUpdate) { + case (updates, (td, blockHash)) => + updates.and(storage.put(blockHash, td)) } + batchUpdates.commit() - blockHashesTdPair.foreach { case (td, blockHash) => assert(totalDifficultyStorage.get(blockHash).contains(td)) } + blockHashesTdPair.foreach { case (td, blockHash) => assert(storage.get(blockHash).contains(td)) } } } @@ -30,23 +31,25 @@ class TotalDifficultyStorageSuite extends FunSuite with ScalaCheckPropertyChecks val blockHashesTdPair = tdList.zip(blockHashes) //Total difficulty of blocks is inserted - val initialTotalDifficultyStorage = new TotalDifficultyStorage(EphemDataSource()) - val totalDifficultyStorage = blockHashesTdPair.foldLeft(initialTotalDifficultyStorage) { - case (recTotalDifficultyStorage, (td, blockHash)) => - recTotalDifficultyStorage.put(blockHash, td) + val storage = new TotalDifficultyStorage(EphemDataSource()) + val storageInsertions = blockHashesTdPair.foldLeft(storage.emptyBatchUpdate) { + case (updates, (td, blockHash)) => + updates.and(storage.put(blockHash, td)) } + storageInsertions.commit() //Total difficulty of blocks is deleted val (toDelete, toLeave) = blockHashesTdPair.splitAt(Gen.choose(0, blockHashesTdPair.size).sample.get) - val totalDifficultyStorageAfterDelete = toDelete.foldLeft(totalDifficultyStorage) { - case (recTotalDifficultyStorage, (_, blockHash)) => - recTotalDifficultyStorage.remove(blockHash) + val storageDeletions = toDelete.foldLeft(storage.emptyBatchUpdate) { + case (updates, (_, blockHash)) => + updates.and(storage.remove(blockHash)) } + storageDeletions.commit() - toLeave.foreach { case (td, blockHeader) => - assert(totalDifficultyStorageAfterDelete.get(blockHeader).contains(td)) + toLeave.foreach { + case (td, blockHeader) => assert(storage.get(blockHeader).contains(td)) } - toDelete.foreach { case (_, bh) => assert(totalDifficultyStorageAfterDelete.get(bh).isEmpty) } + toDelete.foreach { case (_, bh) => assert(storage.get(bh).isEmpty) } } } } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/TransactionMappingStorageSuite.scala b/src/test/scala/io/iohk/ethereum/db/storage/TransactionMappingStorageSuite.scala index ba5bc88f99..f1f5230565 100644 --- a/src/test/scala/io/iohk/ethereum/db/storage/TransactionMappingStorageSuite.scala +++ b/src/test/scala/io/iohk/ethereum/db/storage/TransactionMappingStorageSuite.scala @@ -17,14 +17,15 @@ class TransactionMappingStorageSuite extends FunSuite with ScalaCheckPropertyChe TransactionLocation(blockHash, txIndex) } - val initialTxMappingStorage = new TransactionMappingStorage(EphemDataSource()) - val txMappingStorage = txHashes.zip(txLocationList).foldLeft(initialTxMappingStorage) { - case (recTxMappingStorage, (txHash, txLocation)) => - recTxMappingStorage.put(txHash, txLocation) + val storage = new TransactionMappingStorage(EphemDataSource()) + val batchUpdates = txHashes.zip(txLocationList).foldLeft(storage.emptyBatchUpdate) { + case (updates, (txHash, txLocation)) => + updates.and(storage.put(txHash, txLocation)) } + batchUpdates.commit() - txHashes.zip(txLocationList).foreach { case (txHash, txLocation) => - assert(txMappingStorage.get(txHash).contains(txLocation)) + txHashes.zip(txLocationList).foreach { + case (txHash, txLocation) => assert(storage.get(txHash).contains(txLocation)) } } } @@ -40,22 +41,25 @@ class TransactionMappingStorageSuite extends FunSuite with ScalaCheckPropertyChe val txHashAndLocationPair = txHashes.zip(txLocationList) //Mapping of tx to blocks is inserted - val initialTxMappingStorage = new TransactionMappingStorage(EphemDataSource()) - val txMappingStorage = txHashAndLocationPair.foldLeft(initialTxMappingStorage) { - case (recTxMappingStorage, (txHash, txLocation)) => - recTxMappingStorage.put(txHash, txLocation) + val storage = new TransactionMappingStorage(EphemDataSource()) + val storageInsertions = txHashAndLocationPair.foldLeft(storage.emptyBatchUpdate) { + case (updates, (txHash, txLocation)) => + updates.and(storage.put(txHash, txLocation)) } + storageInsertions.commit() //Mapping of tx to blocks is deleted val (toDelete, toLeave) = txHashAndLocationPair.splitAt(Gen.choose(0, txHashAndLocationPair.size).sample.get) - val txMappingStorageAfterDelete = toDelete.foldLeft(txMappingStorage) { case (recTxMappingStorage, (txHash, _)) => - recTxMappingStorage.remove(txHash) + val storageDeletions = toDelete.foldLeft(storage.emptyBatchUpdate) { + case (updates, (txHash, _)) => + updates.and(storage.remove(txHash)) } + storageDeletions.commit() - toLeave.foreach { case (txHash, txLocation) => - assert(txMappingStorageAfterDelete.get(txHash).contains(txLocation)) + toLeave.foreach { + case (txHash, txLocation) => assert(storage.get(txHash).contains(txLocation)) } - toDelete.foreach { case (txHash, _) => assert(txMappingStorageAfterDelete.get(txHash).isEmpty) } + toDelete.foreach { case (txHash, _) => assert(storage.get(txHash).isEmpty) } } } } diff --git a/src/test/scala/io/iohk/ethereum/db/storage/TransactionalKeyValueStorageSuite.scala b/src/test/scala/io/iohk/ethereum/db/storage/TransactionalKeyValueStorageSuite.scala new file mode 100644 index 0000000000..5b92a98a02 --- /dev/null +++ b/src/test/scala/io/iohk/ethereum/db/storage/TransactionalKeyValueStorageSuite.scala @@ -0,0 +1,127 @@ +package io.iohk.ethereum.db.storage + +import io.iohk.ethereum.ObjectGenerators +import io.iohk.ethereum.db.dataSource.{DataSource, DataSourceUpdate, EphemDataSource} +import io.iohk.ethereum.rlp.RLPImplicits._ +import io.iohk.ethereum.rlp.{decode => rlpDecode, encode => rlpEncode} +import org.scalacheck.Gen +import org.scalatest.FunSuite +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks + +class TransactionalKeyValueStorageSuite extends FunSuite with ScalaCheckPropertyChecks with ObjectGenerators { + val iterationsNumber = 100 + + object IntStorage { + val intNamespace: IndexedSeq[Byte] = IndexedSeq[Byte]('i'.toByte) + val intSerializer: Int => IndexedSeq[Byte] = (i: Int) => rlpEncode(i).toIndexedSeq + val intDeserializer: IndexedSeq[Byte] => Int = + (encodedInt: IndexedSeq[Byte]) => rlpDecode[Int](encodedInt.toArray) + } + + class IntStorage(val dataSource: DataSource) extends TransactionalKeyValueStorage[Int, Int] { + import IntStorage._ + + override val namespace: IndexedSeq[Byte] = intNamespace + override def keySerializer: Int => IndexedSeq[Byte] = intSerializer + override def valueSerializer: Int => IndexedSeq[Byte] = intSerializer + override def valueDeserializer: IndexedSeq[Byte] => Int = intDeserializer + } + + def newIntStorage(): IntStorage = new IntStorage(EphemDataSource()) + + val dataGenerator = for { + intsInStorage <- Gen.nonEmptyListOf(intGen) + intsNotInStorage <- Gen.nonEmptyListOf(intGen.suchThat(value => !intsInStorage.contains(value))) + } yield (intsInStorage, intsNotInStorage) + + test("Get ints from KeyValueStorage") { + forAll(dataGenerator) { case (intsInStorage, intsNotInStorage) => + val intsInStorageIndexedSeq = intsInStorage.map{ IntStorage.intSerializer(_) } + val intDataSource = EphemDataSource() + intDataSource.update( + Seq(DataSourceUpdate(IntStorage.intNamespace, Seq(), intsInStorageIndexedSeq.zip(intsInStorageIndexedSeq))) + ) + val keyValueStorage = new IntStorage(intDataSource) + + intsInStorage.foreach { i => + assert(keyValueStorage.get(i).contains(i)) + } + intsNotInStorage.foreach { i => + assert(keyValueStorage.get(i).isEmpty) + } + } + } + + test("Insert ints to KeyValueStorage") { + forAll(Gen.listOfN(iterationsNumber, Gen.listOf(intGen))) { listOfListOfInt => + val keyValueStorage = newIntStorage() + + listOfListOfInt.foreach { intList => + keyValueStorage.update(Seq(), intList.zip(intList)).commit() + } + + listOfListOfInt.flatten.foreach { i => + assert(keyValueStorage.get(i).contains(i)) + } + } + } + + test("Delete ints from KeyValueStorage") { + forAll(Gen.listOf(intGen)) { listOfInt => + //Insert of keys + val intStorage = newIntStorage() + intStorage.update(Seq(), listOfInt.zip(listOfInt)).commit() + + //Delete of ints + val (toDelete, toLeave) = listOfInt.splitAt(Gen.choose(0, listOfInt.size).sample.get) + intStorage.update(toDelete, Seq()).commit() + + toDelete.foreach { i => + assert(intStorage.get(i).isEmpty) + } + toLeave.foreach { i => + assert(intStorage.get(i).contains(i)) + } + } + } + + test("Put ints into KeyValueStorage") { + forAll(Gen.listOf(intGen)) { listOfInt => + val keyValueStorage = newIntStorage() + + val batchUpdates = listOfInt.foldLeft(keyValueStorage.emptyBatchUpdate) { + case (updates, i) => + updates.and(keyValueStorage.put(i, i)) + } + + batchUpdates.commit() + + listOfInt.foreach { i => + assert(keyValueStorage.get(i).contains(i)) + } + } + } + + test("Remove ints from KeyValueStorage") { + forAll(Gen.listOf(intGen)) { listOfInt => + //Insert of keys + val intStorage = newIntStorage() + intStorage.update(Seq(), listOfInt.zip(listOfInt)).commit() + + //Delete of ints + val (toDelete, toLeave) = listOfInt.splitAt(Gen.choose(0, listOfInt.size).sample.get) + val batchUpdates = toDelete.foldLeft(intStorage.emptyBatchUpdate) { + case (updates, i) => + updates.and(intStorage.remove(i)) + } + batchUpdates.commit() + + toDelete.foreach { i => + assert(intStorage.get(i).isEmpty) + } + toLeave.foreach { i => + assert(intStorage.get(i).contains(i)) + } + } + } +} diff --git a/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala b/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala index fcbdd71d66..d7f95339a7 100644 --- a/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala +++ b/src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala @@ -12,7 +12,7 @@ class BlockchainSpec extends FlatSpec with Matchers { "Blockchain" should "be able to store a block and return if if queried by hash" in new EphemBlockchainTestSetup { val validBlock = Fixtures.Blocks.ValidBlock.block - blockchain.save(validBlock) + blockchain.storeBlock(validBlock).commit() val block = blockchain.getBlockByHash(validBlock.header.hash) assert(block.isDefined) assert(validBlock == block.get) @@ -26,7 +26,7 @@ class BlockchainSpec extends FlatSpec with Matchers { it should "be able to store a block and retrieve it by number" in new EphemBlockchainTestSetup { val validBlock = Fixtures.Blocks.ValidBlock.block - blockchain.save(validBlock) + blockchain.storeBlock(validBlock).commit() val block = blockchain.getBlockByNumber(validBlock.header.number) assert(block.isDefined) assert(validBlock == block.get) @@ -34,7 +34,7 @@ class BlockchainSpec extends FlatSpec with Matchers { it should "be able to query a stored blockHeader by it's number" in new EphemBlockchainTestSetup { val validHeader = Fixtures.Blocks.ValidBlock.header - blockchain.save(validHeader) + blockchain.storeBlockHeader(validHeader).commit() val header = blockchain.getBlockHeaderByNumber(validHeader.number) assert(header.isDefined) assert(validHeader == header.get) @@ -59,7 +59,7 @@ class BlockchainSpec extends FlatSpec with Matchers { val mptWithAcc = emptyMpt.put(address, account) val headerWithAcc = validHeader.copy(stateRoot = ByteString(mptWithAcc.getRootHash)) - blockchain.save(headerWithAcc) + blockchain.storeBlockHeader(headerWithAcc).commit() val retrievedAccount = blockchain.getAccount(address, headerWithAcc.number) retrievedAccount shouldEqual Some(account) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/EthServiceSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/EthServiceSpec.scala index c6ffa248ca..45cd978299 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/EthServiceSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/EthServiceSpec.scala @@ -60,14 +60,14 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getBlockTransactionCountByHash with the block has no tx when the requested block is in the blockchain and has no tx" in new TestSetup { - blockchain.save(blockToRequest.copy(body = BlockBody(Nil, Nil))) + blockchain.storeBlock(blockToRequest.copy(body = BlockBody(Nil, Nil))).commit() val request = TxCountByBlockHashRequest(blockToRequestHash) val response = Await.result(ethService.getBlockTransactionCountByHash(request), Duration.Inf).right.get response.txsQuantity shouldBe Some(0) } it should "answer eth_getBlockTransactionCountByHash correctly when the requested block is in the blockchain and has some tx" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val request = TxCountByBlockHashRequest(blockToRequestHash) val response = Await.result(ethService.getBlockTransactionCountByHash(request), Duration.Inf).right.get response.txsQuantity shouldBe Some(blockToRequest.body.transactionList.size) @@ -82,7 +82,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getTransactionByBlockHashAndIndex with None when there is no tx in requested index" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val invalidTxIndex = blockToRequest.body.transactionList.size val requestWithInvalidIndex = GetTransactionByBlockHashAndIndexRequest(blockToRequest.header.hash, invalidTxIndex) @@ -95,7 +95,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getTransactionByBlockHashAndIndex with the transaction response correctly when the requested index has one" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val txIndexToRequest = blockToRequest.body.transactionList.size / 2 val request = GetTransactionByBlockHashAndIndexRequest(blockToRequest.header.hash, txIndexToRequest) @@ -129,8 +129,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF (ledger.consensus _: (() ⇒ Consensus)).expects().returns(consensus) - blockchain.save(blockToRequest) - blockchain.save(blockToRequestHash, blockTd) + blockchain.storeBlock(blockToRequest) + .and(blockchain.storeTotalDifficulty(blockToRequestHash, blockTd)) + .commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) (blockGenerator.getPendingBlockAndState _).expects().returns(None) @@ -147,8 +148,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getBlockByNumber with the block response correctly when it's totalDifficulty is in blockchain" in new TestSetup { - blockchain.save(blockToRequest) - blockchain.save(blockToRequestHash, blockTd) + blockchain.storeBlock(blockToRequest) + .and(blockchain.storeTotalDifficulty(blockToRequestHash, blockTd)) + .commit() val request = BlockByNumberRequest(BlockParam.WithNumber(blockToRequestNumber), fullTxs = true) val response = Await.result(ethService.getBlockByNumber(request), Duration.Inf).right.get @@ -163,7 +165,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getBlockByNumber with the block response correctly when it's totalDifficulty is not in blockchain" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val request = BlockByNumberRequest(BlockParam.WithNumber(blockToRequestNumber), fullTxs = true) val response = Await.result(ethService.getBlockByNumber(request), Duration.Inf).right.get @@ -178,8 +180,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getBlockByNumber with the block response correctly when the txs should be hashed" in new TestSetup { - blockchain.save(blockToRequest) - blockchain.save(blockToRequestHash, blockTd) + blockchain.storeBlock(blockToRequest) + .and(blockchain.storeTotalDifficulty(blockToRequestHash, blockTd)) + .commit() val request = BlockByNumberRequest(BlockParam.WithNumber(blockToRequestNumber), fullTxs = true) val response = Await.result(ethService.getBlockByNumber(request.copy(fullTxs = false)), Duration.Inf).right.get @@ -196,8 +199,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getBlockByHash with the block response correctly when it's totalDifficulty is in blockchain" in new TestSetup { - blockchain.save(blockToRequest) - blockchain.save(blockToRequestHash, blockTd) + blockchain.storeBlock(blockToRequest) + .and(blockchain.storeTotalDifficulty(blockToRequestHash, blockTd)) + .commit() val request = BlockByBlockHashRequest(blockToRequestHash, fullTxs = true) val response = Await.result(ethService.getByBlockHash(request), Duration.Inf).right.get @@ -212,7 +216,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getBlockByHash with the block response correctly when it's totalDifficulty is not in blockchain" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val request = BlockByBlockHashRequest(blockToRequestHash, fullTxs = true) val response = Await.result(ethService.getByBlockHash(request), Duration.Inf).right.get @@ -227,8 +231,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getBlockByHash with the block response correctly when the txs should be hashed" in new TestSetup { - blockchain.save(blockToRequest) - blockchain.save(blockToRequestHash, blockTd) + blockchain.storeBlock(blockToRequest) + .and(blockchain.storeTotalDifficulty(blockToRequestHash, blockTd)) + .commit() val request = BlockByBlockHashRequest(blockToRequestHash, fullTxs = true) val response = Await.result(ethService.getByBlockHash(request.copy(fullTxs = false)), Duration.Inf).right.get @@ -246,7 +251,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getUncleByBlockHashAndIndex with None when there's no uncle" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val uncleIndexToRequest = 0 val request = UncleByBlockHashAndIndexRequest(blockToRequestHash, uncleIndexToRequest) @@ -256,7 +261,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getUncleByBlockHashAndIndex with None when there's no uncle in the requested index" in new TestSetup { - blockchain.save(blockToRequestWithUncles) + blockchain.storeBlock(blockToRequestWithUncles).commit() val uncleIndexToRequest = 0 val request = UncleByBlockHashAndIndexRequest(blockToRequestHash, uncleIndexToRequest) @@ -268,7 +273,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getUncleByBlockHashAndIndex correctly when the requested index has one but there's no total difficulty for it" in new TestSetup { - blockchain.save(blockToRequestWithUncles) + blockchain.storeBlock(blockToRequestWithUncles).commit() val uncleIndexToRequest = 0 val request = UncleByBlockHashAndIndexRequest(blockToRequestHash, uncleIndexToRequest) @@ -281,8 +286,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "anwer eth_getUncleByBlockHashAndIndex correctly when the requested index has one and there's total difficulty for it" in new TestSetup { - blockchain.save(blockToRequestWithUncles) - blockchain.save(uncle.hash, uncleTd) + blockchain.storeBlock(blockToRequestWithUncles) + .and(blockchain.storeTotalDifficulty(uncle.hash, uncleTd)) + .commit() val uncleIndexToRequest = 0 val request = UncleByBlockHashAndIndexRequest(blockToRequestHash, uncleIndexToRequest) @@ -302,7 +308,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getUncleByBlockNumberAndIndex with None when there's no uncle" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val uncleIndexToRequest = 0 val request = UncleByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequestNumber), uncleIndexToRequest) @@ -312,7 +318,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getUncleByBlockNumberAndIndex with None when there's no uncle in the requested index" in new TestSetup { - blockchain.save(blockToRequestWithUncles) + blockchain.storeBlock(blockToRequestWithUncles).commit() val uncleIndexToRequest = 0 val request = UncleByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequestNumber), uncleIndexToRequest) @@ -324,7 +330,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "answer eth_getUncleByBlockNumberAndIndex correctly when the requested index has one but there's no total difficulty for it" in new TestSetup { - blockchain.save(blockToRequestWithUncles) + blockchain.storeBlock(blockToRequestWithUncles).commit() val uncleIndexToRequest = 0 val request = UncleByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequestNumber), uncleIndexToRequest) @@ -337,8 +343,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "anwer eth_getUncleByBlockNumberAndIndex correctly when the requested index has one and there's total difficulty for it" in new TestSetup { - blockchain.save(blockToRequestWithUncles) - blockchain.save(uncle.hash, uncleTd) + blockchain.storeBlock(blockToRequestWithUncles) + .and(blockchain.storeTotalDifficulty(uncle.hash, uncleTd)) + .commit() val uncleIndexToRequest = 0 val request = UncleByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequestNumber), uncleIndexToRequest) @@ -419,7 +426,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "execute call and return a value" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) val txResult = TxResult(BlockchainImpl(storagesInstance.storages).getWorldStateProxy(-1, UInt256.Zero, None, @@ -436,7 +443,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "execute estimateGas and return a value" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) val estimatedGas = BigInt(123) @@ -452,7 +459,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "get uncle count by block number" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) val response = ethService.getUncleCountByBlockNumber(GetUncleCountByBlockNumberRequest(BlockParam.Latest)) @@ -461,7 +468,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "get uncle count by block hash" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val response = ethService.getUncleCountByBlockHash(GetUncleCountByBlockHashRequest(blockToRequest.header.hash)) @@ -469,7 +476,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "get transaction count by block number" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val response = ethService.getBlockTransactionCountByNumber(GetBlockTransactionCountByNumberRequest(BlockParam.WithNumber(blockToRequest.header.number))) @@ -477,7 +484,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "get transaction count by latest block number" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) val response = ethService.getBlockTransactionCountByNumber(GetBlockTransactionCountByNumberRequest(BlockParam.Latest)) @@ -487,7 +494,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF it should "handle getCode request" in new TestSetup { val address = Address(ByteString(Hex.decode("abbb6bebfa05aa13e908eaa492bd7a8343760477"))) - storagesInstance.storages.evmCodeStorage.put(ByteString("code hash"), ByteString("code code code")) + storagesInstance.storages.evmCodeStorage.put(ByteString("code hash"), ByteString("code code code")).commit() import MerklePatriciaTrie.defaultByteArraySerializable @@ -497,7 +504,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) - blockchain.save(newblock) + blockchain.storeBlock(newblock).commit() blockchain.saveBestKnownBlock(newblock.header.number) val response = ethService.getCode(GetCodeRequest(address, BlockParam.Latest)) @@ -543,7 +550,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF ethService.getMining(GetMiningRequest()).futureValue shouldEqual Right(GetMiningResponse(false)) (blockGenerator.generateBlock _).expects(parentBlock, *, *, *).returning(Right(PendingBlock(block, Nil))) - blockchain.save(parentBlock) + blockchain.storeBlock(parentBlock).commit() ethService.getWork(GetWorkRequest()) val response = ethService.getMining(GetMiningRequest()) @@ -581,7 +588,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF (ledger.consensus _: (() ⇒ Consensus)).expects().returns(consensus).anyNumberOfTimes() (blockGenerator.generateBlock _).expects(parentBlock, *, *, *).returning(Right(PendingBlock(block, Nil))) - blockchain.save(parentBlock) + blockchain.storeBlock(parentBlock).commit() ethService.getWork(GetWorkRequest()) Thread.sleep(minerActiveTimeout.toMillis) @@ -607,14 +614,14 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF it should "return average gas price" in new TestSetup { blockchain.saveBestKnownBlock(42) - blockchain.save(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)) + blockchain.storeBlock(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)).commit() val response = ethService.getGetGasPrice(GetGasPriceRequest()) response.futureValue shouldEqual Right(GetGasPriceResponse(BigInt("20000000000"))) } it should "getTransactionByBlockNumberAndIndexRequest return transaction by index" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) val txIndex: Int = 1 @@ -626,7 +633,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "getTransactionByBlockNumberAndIndexRequest return empty response if transaction does not exists when getting by index" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val txIndex: Int = blockToRequest.body.transactionList.length + 42 val request = GetTransactionByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequest.header.number), txIndex) @@ -636,7 +643,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF } it should "getTransactionByBlockNumberAndIndexRequest return empty response if block does not exists when getting by index" in new TestSetup { - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val txIndex: Int = 1 val request = GetTransactionByBlockNumberAndIndexRequest(BlockParam.WithNumber(blockToRequest.header.number - 42), txIndex) @@ -656,7 +663,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) - blockchain.save(newblock) + blockchain.storeBlock(newblock).commit() blockchain.saveBestKnownBlock(newblock.header.number) @@ -692,7 +699,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) - blockchain.save(newblock) + blockchain.storeBlock(newblock).commit() blockchain.saveBestKnownBlock(newblock.header.number) val response = ethService.getStorageAt(GetStorageAtRequest(address, 333, BlockParam.Latest)) @@ -710,7 +717,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF val newBlockHeader = blockToRequest.header.copy(stateRoot = ByteString(mpt.getRootHash)) val newblock = blockToRequest.copy(header = newBlockHeader) - blockchain.save(newblock) + blockchain.storeBlock(newblock).commit() blockchain.saveBestKnownBlock(newblock.header.number) val response = ethService.getTransactionCount(GetTransactionCountRequest(address, BlockParam.Latest)) @@ -746,7 +753,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF (ledger.consensus _: (() ⇒ Consensus)).expects().returns(consensus) val blockWithTx = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) - blockchain.save(blockWithTx) + blockchain.storeBlock(blockWithTx).commit() val request = GetTransactionByHashRequest(txToRequestHash) val response = ethService.getTransactionByHash(request) @@ -761,10 +768,11 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF it should "calculate correct contract address for contract creating by transaction" in new TestSetup { val body = BlockBody(Seq(Fixtures.Blocks.Block3125369.body.transactionList.head, contractCreatingTransaction), Nil) val blockWithTx = Block(Fixtures.Blocks.Block3125369.header, body) - blockchain.save(blockWithTx) val gasUsedByTx = 4242 - blockchain.save(Fixtures.Blocks.Block3125369.header.hash, - Seq(fakeReceipt, fakeReceipt.copy(cumulativeGasUsed = fakeReceipt.cumulativeGasUsed + gasUsedByTx))) + blockchain.storeBlock(blockWithTx) + .and(blockchain.storeReceipts(Fixtures.Blocks.Block3125369.header.hash, + Seq(fakeReceipt, fakeReceipt.copy(cumulativeGasUsed = fakeReceipt.cumulativeGasUsed + gasUsedByTx)))) + .commit() val request = GetTransactionReceiptRequest(contractCreatingTransaction.hash) val response = ethService.getTransactionReceipt(request) @@ -807,8 +815,9 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF val blockWithTxs2and3 = Block(Fixtures.Blocks.Block3125369.header.copy(number = 3125370), Fixtures.Blocks.Block3125369.body.copy( transactionList = Seq(tx2, tx3))) - blockchain.save(blockWithTx1) - blockchain.save(blockWithTxs2and3) + blockchain.storeBlock(blockWithTx1) + .and(blockchain.storeBlock(blockWithTxs2and3)) + .commit() val request = GetAccountTransactionsRequest(address, 3125360, 3125370) @@ -840,7 +849,7 @@ class EthServiceSpec extends FlatSpec with Matchers with ScalaFutures with MockF (ledger.consensus _: (() ⇒ Consensus)).expects().returns(consensus) val blockWithTx = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) - blockchain.save(blockWithTx) + blockchain.storeBlock(blockWithTx).commit() val keyPair = crypto.generateKeyPair(new SecureRandom) diff --git a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerSpec.scala b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerSpec.scala index 15ff0375b4..47d4f0b500 100644 --- a/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerSpec.scala +++ b/src/test/scala/io/iohk/ethereum/jsonrpc/JsonRpcControllerSpec.scala @@ -192,7 +192,7 @@ class JsonRpcControllerSpec it should "handle eth_getBlockTransactionCountByHash request" in new TestSetup { val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val rpcRequest = JsonRpcRequest( "2.0", @@ -215,8 +215,9 @@ class JsonRpcControllerSpec val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) val blockTd = blockToRequest.header.difficulty - blockchain.save(blockToRequest) - blockchain.save(blockToRequest.header.hash, blockTd) + blockchain.storeBlock(blockToRequest) + .and(blockchain.storeTotalDifficulty(blockToRequest.header.hash, blockTd)) + .commit() val request = JsonRpcRequest( "2.0", @@ -240,8 +241,9 @@ class JsonRpcControllerSpec val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) val blockTd = blockToRequest.header.difficulty - blockchain.save(blockToRequest) - blockchain.save(blockToRequest.header.hash, blockTd) + blockchain.storeBlock(blockToRequest) + .and(blockchain.storeTotalDifficulty(blockToRequest.header.hash, blockTd)) + .commit() val request = JsonRpcRequest( "2.0", @@ -264,7 +266,7 @@ class JsonRpcControllerSpec val uncle = Fixtures.Blocks.DaoForkBlock.header val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, BlockBody(Nil, Seq(uncle))) - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val request: JsonRpcRequest = JsonRpcRequest( "2.0", @@ -298,7 +300,7 @@ class JsonRpcControllerSpec val uncle = Fixtures.Blocks.DaoForkBlock.header val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, BlockBody(Nil, Seq(uncle))) - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val request: JsonRpcRequest = JsonRpcRequest( "2.0", @@ -332,7 +334,7 @@ class JsonRpcControllerSpec val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) val txIndexToRequest = blockToRequest.body.transactionList.size / 2 - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) val request: JsonRpcRequest = JsonRpcRequest( @@ -810,7 +812,7 @@ class JsonRpcControllerSpec } it should "eth_gasPrice" in new TestSetup { - blockchain.save(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)) + blockchain.storeBlock(Block(Fixtures.Blocks.Block3125369.header.copy(number = 42), Fixtures.Blocks.Block3125369.body)).commit() blockchain.saveBestKnownBlock(42) val request: JsonRpcRequest = JsonRpcRequest( @@ -1093,7 +1095,7 @@ class JsonRpcControllerSpec val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) val txIndex = 1 - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() blockchain.saveBestKnownBlock(blockToRequest.header.number) val request: JsonRpcRequest = JsonRpcRequest( @@ -1126,7 +1128,7 @@ class JsonRpcControllerSpec Block(Fixtures.Blocks.Block3125369.header.copy(number = BigInt(0xc005)), Fixtures.Blocks.Block3125369.body) val txIndex = 1 - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val request: JsonRpcRequest = JsonRpcRequest( "2.0", @@ -1157,7 +1159,7 @@ class JsonRpcControllerSpec val blockToRequest = Block(Fixtures.Blocks.Block3125369.header, Fixtures.Blocks.Block3125369.body) val txIndex = 1 - blockchain.save(blockToRequest) + blockchain.storeBlock(blockToRequest).commit() val request: JsonRpcRequest = JsonRpcRequest( "2.0", diff --git a/src/test/scala/io/iohk/ethereum/ledger/LedgerTestSetup.scala b/src/test/scala/io/iohk/ethereum/ledger/LedgerTestSetup.scala index d86024c8bf..87df7f3f68 100644 --- a/src/test/scala/io/iohk/ethereum/ledger/LedgerTestSetup.scala +++ b/src/test/scala/io/iohk/ethereum/ledger/LedgerTestSetup.scala @@ -149,10 +149,11 @@ trait BlockchainSetup extends TestSetup { ) val validBlockBodyWithNoTxs: BlockBody = BlockBody(Nil, Nil) - blockchain.save(validBlockParentHeader) - blockchain.save(validBlockParentHeader.hash, validBlockBodyWithNoTxs) - storagesInstance.storages.appStateStorage.putBestBlockNumber(validBlockParentHeader.number) - storagesInstance.storages.totalDifficultyStorage.put(validBlockParentHeader.hash, 0) + blockchain.storeBlockHeader(validBlockParentHeader) + .and(blockchain.storeBlockBody(validBlockParentHeader.hash, validBlockBodyWithNoTxs)) + .and(storagesInstance.storages.appStateStorage.putBestBlockNumber(validBlockParentHeader.number)) + .and(storagesInstance.storages.totalDifficultyStorage.put(validBlockParentHeader.hash, 0)) + .commit() val validTx: Transaction = defaultTx.copy( nonce = initialOriginNonce, diff --git a/src/test/scala/io/iohk/ethereum/ledger/StxLedgerSpec.scala b/src/test/scala/io/iohk/ethereum/ledger/StxLedgerSpec.scala index f3628c9abf..b4c4e578bf 100644 --- a/src/test/scala/io/iohk/ethereum/ledger/StxLedgerSpec.scala +++ b/src/test/scala/io/iohk/ethereum/ledger/StxLedgerSpec.scala @@ -184,7 +184,8 @@ trait ScenarioSetup extends EphemBlockchainTestSetup { val genesisHeader: BlockHeader = genesisBlock.header val lastBlockGasLimit: BigInt = genesisBlock.header.gasLimit - blockchain.save(genesisBlock) - blockchain.save(genesisHash, Nil) - blockchain.save(genesisHash, genesisBlock.header.difficulty) + blockchain.storeBlock(genesisBlock) + .and(blockchain.storeReceipts(genesisHash, Nil)) + .and(blockchain.storeTotalDifficulty(genesisHash, genesisBlock.header.difficulty)) + .commit() } diff --git a/src/test/scala/io/iohk/ethereum/mpt/MerklePatriciaTrieSuite.scala b/src/test/scala/io/iohk/ethereum/mpt/MerklePatriciaTrieSuite.scala index 63e33fea5b..da14fe9698 100644 --- a/src/test/scala/io/iohk/ethereum/mpt/MerklePatriciaTrieSuite.scala +++ b/src/test/scala/io/iohk/ethereum/mpt/MerklePatriciaTrieSuite.scala @@ -4,7 +4,7 @@ import java.nio.ByteBuffer import java.security.MessageDigest import akka.util.ByteString import io.iohk.ethereum.ObjectGenerators -import io.iohk.ethereum.db.dataSource.EphemDataSource +import io.iohk.ethereum.db.dataSource.{DataSourceUpdate, EphemDataSource} import io.iohk.ethereum.db.storage._ import io.iohk.ethereum.db.storage.pruning.BasicPruning import io.iohk.ethereum.mpt.MerklePatriciaTrie.{MPTException, defaultByteArraySerializable} @@ -342,12 +342,15 @@ class MerklePatriciaTrieSuite extends FunSuite with ScalaCheckPropertyChecks wit val key4: Array[Byte] = Hex.decode("123500") val trie = EmptyTrie.put(key1, key1).put(key2, key2).put(key3, key3).put(key4, key4) val wrongSource = EphemDataSource() - .update( - IndexedSeq[Byte]('e'.toByte), - toRemove = Seq(), - toUpsert = Seq(ByteString(trie.getRootHash) -> trie.nodeStorage.get(trie.getRootHash).cachedRlpEncoded.get) + wrongSource.update( + Seq( + DataSourceUpdate( + IndexedSeq[Byte]('e'.toByte), + toRemove = Seq(), + toUpsert = Seq(ByteString(trie.getRootHash) -> trie.nodeStorage.get(trie.getRootHash).cachedRlpEncoded.get) + ) ) - .asInstanceOf[EphemDataSource] + ) val trieAfterDelete = Try { val trieWithWrongSource = MerklePatriciaTrie[Array[Byte], Array[Byte]](trie.getRootHash, StateStorage.getReadOnlyStorage(wrongSource)) diff --git a/src/test/scala/io/iohk/ethereum/network/EtcPeerManagerSpec.scala b/src/test/scala/io/iohk/ethereum/network/EtcPeerManagerSpec.scala index 3866ed20ac..89a9b00cfc 100644 --- a/src/test/scala/io/iohk/ethereum/network/EtcPeerManagerSpec.scala +++ b/src/test/scala/io/iohk/ethereum/network/EtcPeerManagerSpec.scala @@ -267,7 +267,7 @@ class EtcPeerManagerSpec extends FlatSpec with Matchers { trait TestSetup extends EphemBlockchainTestSetup { override implicit lazy val system = ActorSystem("PeersInfoHolderSpec_System") - blockchain.save(Fixtures.Blocks.Genesis.header) + blockchain.storeBlockHeader(Fixtures.Blocks.Genesis.header).commit() override lazy val blockchainConfig = Config.blockchains.blockchainConfig val forkResolver = new ForkResolver.EtcForkResolver(blockchainConfig.daoForkConfig.get) diff --git a/src/test/scala/io/iohk/ethereum/network/p2p/PeerActorSpec.scala b/src/test/scala/io/iohk/ethereum/network/p2p/PeerActorSpec.scala index a8db3e7d33..3646e0130e 100644 --- a/src/test/scala/io/iohk/ethereum/network/p2p/PeerActorSpec.scala +++ b/src/test/scala/io/iohk/ethereum/network/p2p/PeerActorSpec.scala @@ -189,8 +189,9 @@ class PeerActorSpec extends FlatSpec with Matchers { val header = Fixtures.Blocks.ValidBlock.header.copy(difficulty = daoForkBlockTotalDifficulty + 100000, number = 3000000) storagesInstance.storages.appStateStorage.putBestBlockNumber(3000000) // after the fork - blockchain.save(header) - storagesInstance.storages.blockNumberMappingStorage.put(3000000, header.hash) + .and(blockchain.storeBlockHeader(header)) + .and(storagesInstance.storages.blockNumberMappingStorage.put(3000000, header.hash)) + .commit() val remoteStatus = Status( protocolVersion = Versions.PV63, @@ -249,7 +250,7 @@ class PeerActorSpec extends FlatSpec with Matchers { it should "respond to fork block request during the handshake" in new TestSetup { //Save dao fork block - blockchain.save(Fixtures.Blocks.DaoForkBlock.header) + blockchain.storeBlockHeader(Fixtures.Blocks.DaoForkBlock.header).commit() //Handshake till EtcForkBlockExchangeState peer ! PeerActor.ConnectTo(new URI("encode://localhost:9000"))