[ETCM-135] Tx history improvements for wallet (#808)

Author: kapke

insomnia_workspace.json

@@ -25,7 +25,14 @@ import io.iohk.ethereum.network.p2p.EthereumMessageDecoder
 import io.iohk.ethereum.network.p2p.messages.CommonMessages.NewBlock
 import io.iohk.ethereum.network.rlpx.AuthHandshaker
 import io.iohk.ethereum.network.rlpx.RLPxConnectionHandler.RLPxConfiguration
-import io.iohk.ethereum.network.{EtcPeerManagerActor, ForkResolver, KnownNodesManager, PeerEventBusActor, PeerManagerActor, ServerActor}
+import io.iohk.ethereum.network.{
+  EtcPeerManagerActor,
+  ForkResolver,
+  KnownNodesManager,
+  PeerEventBusActor,
+  PeerManagerActor,
+  ServerActor
+}
 import io.iohk.ethereum.nodebuilder.{PruningConfigBuilder, SecureRandomBuilder}
 import io.iohk.ethereum.sync.util.SyncCommonItSpec._
 import io.iohk.ethereum.sync.util.SyncCommonItSpecUtils._

src/it/scala/io/iohk/ethereum/sync/util/CommonFakePeer.scala

@@ -207,11 +207,11 @@ mantis {
       }
 
       # Enabled JSON-RPC APIs over the JSON-RPC endpoint
-      # Available choices are: web3, eth, net, personal, daedalus, test, iele, debug, qa, checkpointing
-      apis = "eth,web3,net,personal,daedalus,debug,qa,checkpointing"
+      # Available choices are: web3, eth, net, personal, mantis, test, iele, debug, qa, checkpointing
+      apis = "eth,web3,net,personal,mantis,debug,qa,checkpointing"
 
-      # Maximum number of blocks for daedalus_getAccountTransactions
-      account-transactions-max-blocks = 50000
+      # Maximum number of blocks for mantis_getAccountTransactions
+      account-transactions-max-blocks = 1000
 
       net {
         peer-manager-timeout = 5.seconds

src/main/resources/application.conf

@@ -4,6 +4,7 @@ import akka.util.ByteString
 import io.iohk.ethereum.jsonrpc.EthService._
 import io.iohk.ethereum.jsonrpc.JsonRpcError.InvalidParams
 import io.iohk.ethereum.jsonrpc.PersonalService.{SendTransactionRequest, SendTransactionResponse, SignRequest}
+import io.iohk.ethereum.jsonrpc.serialization.JsonEncoder.OptionToNull._
 import io.iohk.ethereum.jsonrpc.serialization.JsonMethodDecoder.NoParamsMethodDecoder
 import io.iohk.ethereum.jsonrpc.serialization.{JsonEncoder, JsonMethodCodec, JsonMethodDecoder}
 import org.json4s.JsonAST.{JArray, JBool, JString, JValue, _}
@@ -12,6 +13,7 @@ import org.json4s.{Extraction, JsonAST}
 
 // scalastyle:off number.of.methods
 object EthJsonMethodsImplicits extends JsonMethodsImplicits {
+  implicit val transactionResponseJsonEncoder: JsonEncoder[TransactionResponse] = Extraction.decompose(_)
 
   implicit val eth_protocolVersion = new NoParamsMethodDecoder(ProtocolVersionRequest())
     with JsonEncoder[ProtocolVersionResponse] {
@@ -148,7 +150,7 @@ object EthJsonMethodsImplicits extends JsonMethodsImplicits {
         }
 
       override def encodeJson(t: GetTransactionByHashResponse): JValue =
-        Extraction.decompose(t.txResponse)
+        JsonEncoder.encode(t.txResponse)
     }
 
   implicit val eth_getTransactionReceipt =
@@ -169,7 +171,7 @@ object EthJsonMethodsImplicits extends JsonMethodsImplicits {
   implicit val GetTransactionByBlockHashAndIndexResponseEncoder =
     new JsonEncoder[GetTransactionByBlockHashAndIndexResponse] {
       override def encodeJson(t: GetTransactionByBlockHashAndIndexResponse): JValue =
-        t.transactionResponse.map(Extraction.decompose).getOrElse(JNull)
+        JsonEncoder.encode(t.transactionResponse)
     }
 
   implicit val GetTransactionByBlockHashAndIndexRequestDecoder =
@@ -188,7 +190,7 @@ object EthJsonMethodsImplicits extends JsonMethodsImplicits {
   implicit val GetTransactionByBlockNumberAndIndexResponseEncoder =
     new JsonEncoder[GetTransactionByBlockNumberAndIndexResponse] {
       override def encodeJson(t: GetTransactionByBlockNumberAndIndexResponse): JValue =
-        t.transactionResponse.map(Extraction.decompose).getOrElse(JNull)
+        JsonEncoder.encode(t.transactionResponse)
     }
 
   implicit val GetTransactionByBlockNumberAndIndexRequestDecoder =
@@ -575,23 +577,6 @@ object EthJsonMethodsImplicits extends JsonMethodsImplicits {
     )
   }
 
-  implicit val daedalus_getAccountTransactions =
-    new JsonMethodDecoder[GetAccountTransactionsRequest] with JsonEncoder[GetAccountTransactionsResponse] {
-      def decodeJson(params: Option[JArray]): Either[JsonRpcError, GetAccountTransactionsRequest] =
-        params match {
-          case Some(JArray(JString(addrJson) :: fromBlockJson :: toBlockJson :: Nil)) =>
-            for {
-              addr <- extractAddress(addrJson)
-              fromBlock <- extractQuantity(fromBlockJson)
-              toBlock <- extractQuantity(toBlockJson)
-            } yield GetAccountTransactionsRequest(addr, fromBlock, toBlock)
-          case _ => Left(InvalidParams())
-        }
-
-      override def encodeJson(t: GetAccountTransactionsResponse): JValue =
-        JObject("transactions" -> JArray(t.transactions.map(Extraction.decompose).toList))
-    }
-
   implicit val eth_getStorageRoot = new JsonMethodDecoder[GetStorageRootRequest]
     with JsonEncoder[GetStorageRootResponse] {
     def decodeJson(params: Option[JArray]): Either[JsonRpcError, GetStorageRootRequest] =

src/main/scala/io/iohk/ethereum/jsonrpc/EthJsonMethodsImplicits.scala

@@ -1,8 +1,5 @@
 package io.iohk.ethereum.jsonrpc
 
-import java.time.Duration
-import java.util.Date
-import java.util.concurrent.atomic.AtomicReference
 import akka.actor.ActorRef
 import akka.util.{ByteString, Timeout}
 import cats.syntax.either._
@@ -15,8 +12,10 @@ import io.iohk.ethereum.consensus.ethash.EthashUtils
 import io.iohk.ethereum.crypto._
 import io.iohk.ethereum.db.storage.TransactionMappingStorage.TransactionLocation
 import io.iohk.ethereum.domain.{BlockHeader, SignedTransaction, UInt256, _}
+import io.iohk.ethereum.jsonrpc.AkkaTaskOps._
 import io.iohk.ethereum.jsonrpc.FilterManager.{FilterChanges, FilterLogs, LogFilterLogs, TxLog}
 import io.iohk.ethereum.jsonrpc.server.controllers.JsonRpcBaseController.JsonRpcConfig
+import io.iohk.ethereum.jsonrpc.{FilterManager => FM}
 import io.iohk.ethereum.keystore.KeyStore
 import io.iohk.ethereum.ledger.{InMemoryWorldStateProxy, Ledger, StxLedger}
 import io.iohk.ethereum.mpt.MerklePatriciaTrie.MissingNodeException
@@ -29,10 +28,12 @@ import io.iohk.ethereum.rlp.UInt256RLPImplicits._
 import io.iohk.ethereum.transactions.PendingTransactionsManager
 import io.iohk.ethereum.transactions.PendingTransactionsManager.{PendingTransaction, PendingTransactionsResponse}
 import io.iohk.ethereum.utils._
-import io.iohk.ethereum.jsonrpc.AkkaTaskOps._
-import io.iohk.ethereum.jsonrpc.{FilterManager => FM}
 import monix.eval.Task
 import org.bouncycastle.util.encoders.Hex
+
+import java.time.Duration
+import java.util.Date
+import java.util.concurrent.atomic.AtomicReference
 import scala.collection.concurrent.{TrieMap, Map => ConcurrentMap}
 import scala.concurrent.duration.FiniteDuration
 import scala.language.existentials
@@ -78,9 +79,6 @@ object EthService {
   case class GetTransactionByHashRequest(txHash: ByteString)
   case class GetTransactionByHashResponse(txResponse: Option[TransactionResponse])
 
-  case class GetAccountTransactionsRequest(address: Address, fromBlock: BigInt, toBlock: BigInt)
-  case class GetAccountTransactionsResponse(transactions: Seq[TransactionResponse])
-
   case class GetTransactionReceiptRequest(txHash: ByteString)
   case class GetTransactionReceiptResponse(txResponse: Option[TransactionReceiptResponse])
 
@@ -325,7 +323,7 @@ class EthService(
   }
 
   def getTransactionDataByHash(txHash: ByteString): Task[Option[TransactionData]] = {
-    val maybeTxPendingResponse: Task[Option[TransactionData]] = getTransactionsFromPool().map {
+    val maybeTxPendingResponse: Task[Option[TransactionData]] = getTransactionsFromPool.map {
       _.pendingTransactions.map(_.stx.tx).find(_.hash == txHash).map(TransactionData(_))
     }
 
@@ -541,7 +539,7 @@ class EthService(
       reportActive()
       val bestBlock = blockchain.getBestBlock()
       val response: ServiceResponse[GetWorkResponse] =
-        Task.parZip2(getOmmersFromPool(bestBlock.hash), getTransactionsFromPool()).map { case (ommers, pendingTxs) =>
+        Task.parZip2(getOmmersFromPool(bestBlock.hash), getTransactionsFromPool).map { case (ommers, pendingTxs) =>
           val blockGenerator = ethash.blockGenerator
           val PendingBlockAndState(pb, _) = blockGenerator.generateBlock(
             bestBlock,
@@ -575,13 +573,13 @@ class EthService(
     })(Task.now(OmmersPool.Ommers(Nil))) // NOTE If not Ethash consensus, ommers do not make sense, so => Nil
 
   // TODO This seems to be re-implemented in TransactionPicker, probably move to a better place? Also generalize the error message.
-  private[jsonrpc] def getTransactionsFromPool(): Task[PendingTransactionsResponse] = {
+  private[jsonrpc] val getTransactionsFromPool: Task[PendingTransactionsResponse] = {
     implicit val timeout: Timeout = Timeout(getTransactionFromPoolTimeout)
 
     pendingTransactionsManager
       .askFor[PendingTransactionsResponse](PendingTransactionsManager.GetPendingTransactions)
       .onErrorRecoverWith { case ex: Throwable =>
-        log.error("failed to get transactions, mining block with empty transactions list", ex)
+        log.error("Failed to get pending transactions, passing empty transactions list", ex)
         Task.now(PendingTransactionsResponse(Nil))
       }
   }
@@ -623,8 +621,8 @@ class EthService(
                 startingBlock = startingBlockNumber,
                 currentBlock = blocksProgress.current,
                 highestBlock = blocksProgress.target,
-                knownStates = stateNodesProgress.current,
-                pulledStates = stateNodesProgress.target
+                knownStates = stateNodesProgress.target,
+                pulledStates = stateNodesProgress.current
               )
             )
           )
@@ -927,47 +925,6 @@ class EthService(
     }
   }
 
-  def getAccountTransactions(
-      request: GetAccountTransactionsRequest
-  ): ServiceResponse[GetAccountTransactionsResponse] = {
-    val numBlocksToSearch = request.toBlock - request.fromBlock
-    if (numBlocksToSearch > jsonRpcConfig.accountTransactionsMaxBlocks) {
-      Task.now(
-        Left(
-          JsonRpcError.InvalidParams(
-            s"""Maximum number of blocks to search is ${jsonRpcConfig.accountTransactionsMaxBlocks}, requested: $numBlocksToSearch.
-           |See: 'network.rpc.account-transactions-max-blocks' config.""".stripMargin
-          )
-        )
-      )
-    } else {
-
-      def collectTxs(
-          blockHeader: Option[BlockHeader],
-          pending: Boolean
-      ): PartialFunction[SignedTransaction, TransactionResponse] = {
-        case stx if stx.safeSenderIsEqualTo(request.address) =>
-          TransactionResponse(stx, blockHeader, pending = Some(pending), isOutgoing = Some(true))
-        case stx if stx.tx.receivingAddress.contains(request.address) =>
-          TransactionResponse(stx, blockHeader, pending = Some(pending), isOutgoing = Some(false))
-      }
-
-      getTransactionsFromPool map { case PendingTransactionsResponse(pendingTransactions) =>
-        val pendingTxs = pendingTransactions
-          .map(_.stx.tx)
-          .collect(collectTxs(None, pending = true))
-
-        val txsFromBlocks = (request.toBlock to request.fromBlock by -1).toStream
-          .flatMap { n => blockchain.getBlockByNumber(n) }
-          .flatMap { block =>
-            block.body.transactionList.collect(collectTxs(Some(block.header), pending = false)).reverse
-          }
-
-        Right(GetAccountTransactionsResponse(pendingTxs ++ txsFromBlocks))
-      }
-    }
-  }
-
   def getStorageRoot(req: GetStorageRootRequest): ServiceResponse[GetStorageRootResponse] =
     withAccount(req.address, req.block) { account =>
       GetStorageRootResponse(account.storageRoot)
@@ -980,7 +937,7 @@ class EthService(
     * @return pending transactions
     */
   def ethPendingTransactions(req: EthPendingTransactionsRequest): ServiceResponse[EthPendingTransactionsResponse] =
-    getTransactionsFromPool().map { resp =>
+    getTransactionsFromPool.map { resp =>
       Right(EthPendingTransactionsResponse(resp.pendingTransactions))
     }
 }

src/main/scala/io/iohk/ethereum/jsonrpc/EthService.scala

@@ -22,13 +22,13 @@ import scala.util.Try
 trait JsonMethodsImplicits {
   implicit val formats = JsonSerializers.formats
 
-  protected def encodeAsHex(input: ByteString): JString =
+  def encodeAsHex(input: ByteString): JString =
     JString(s"0x${Hex.toHexString(input.toArray[Byte])}")
 
-  protected def encodeAsHex(input: Byte): JString =
+  def encodeAsHex(input: Byte): JString =
     JString(s"0x${Hex.toHexString(Array(input))}")
 
-  protected def encodeAsHex(input: BigInt): JString =
+  def encodeAsHex(input: BigInt): JString =
     JString(s"0x${input.toString(16)}")
 
   protected def decode(s: String): Array[Byte] = {

src/main/scala/io/iohk/ethereum/jsonrpc/JsonMethodsImplicits.scala

@@ -3,6 +3,7 @@ package io.iohk.ethereum.jsonrpc
 import io.iohk.ethereum.jsonrpc.CheckpointingService._
 import io.iohk.ethereum.jsonrpc.DebugService.{ListPeersInfoRequest, ListPeersInfoResponse}
 import io.iohk.ethereum.jsonrpc.EthService._
+import io.iohk.ethereum.jsonrpc.MantisService.{GetAccountTransactionsRequest, GetAccountTransactionsResponse}
 import io.iohk.ethereum.jsonrpc.NetService._
 import io.iohk.ethereum.jsonrpc.PersonalService._
 import io.iohk.ethereum.jsonrpc.QAService.{
@@ -29,6 +30,7 @@ class JsonRpcController(
     debugService: DebugService,
     qaService: QAService,
     checkpointingService: CheckpointingService,
+    mantisService: MantisService,
     override val config: JsonRpcConfig
 ) extends ApisBuilder
     with Logger
@@ -41,13 +43,14 @@ class JsonRpcController(
   import JsonMethodsImplicits._
   import QAJsonMethodsImplicits._
   import TestJsonMethodsImplicits._
+  import MantisJsonMethodImplicits._
 
   override def apisHandleFns: Map[String, PartialFunction[JsonRpcRequest, Task[JsonRpcResponse]]] = Map(
     Apis.Eth -> handleEthRequest,
     Apis.Web3 -> handleWeb3Request,
     Apis.Net -> handleNetRequest,
     Apis.Personal -> handlePersonalRequest,
-    Apis.Daedalus -> handleDaedalusRequest,
+    Apis.Mantis -> handleMantisRequest,
     Apis.Rpc -> handleRpcRequest,
     Apis.Debug -> handleDebugRequest,
     Apis.Test -> handleTestRequest,
@@ -259,9 +262,9 @@ class JsonRpcController(
       handle[EcRecoverRequest, EcRecoverResponse](personalService.ecRecover, req)
   }
 
-  private def handleDaedalusRequest: PartialFunction[JsonRpcRequest, Task[JsonRpcResponse]] = {
-    case req @ JsonRpcRequest(_, "daedalus_getAccountTransactions", _, _) =>
-      handle[GetAccountTransactionsRequest, GetAccountTransactionsResponse](ethService.getAccountTransactions, req)
+  private def handleMantisRequest: PartialFunction[JsonRpcRequest, Task[JsonRpcResponse]] = {
+    case req @ JsonRpcRequest(_, "mantis_getAccountTransactions", _, _) =>
+      handle[GetAccountTransactionsRequest, GetAccountTransactionsResponse](mantisService.getAccountTransactions, req)
   }
 
   private def handleQARequest: PartialFunction[JsonRpcRequest, Task[JsonRpcResponse]] = {

src/main/scala/io/iohk/ethereum/jsonrpc/JsonRpcController.scala

@@ -0,0 +1,49 @@
+package io.iohk.ethereum.jsonrpc
+
+import io.iohk.ethereum.jsonrpc.EthJsonMethodsImplicits.transactionResponseJsonEncoder
+import io.iohk.ethereum.jsonrpc.JsonRpcError.InvalidParams
+import io.iohk.ethereum.jsonrpc.MantisService.{GetAccountTransactionsRequest, GetAccountTransactionsResponse}
+import io.iohk.ethereum.jsonrpc.serialization.JsonEncoder.Ops._
+import io.iohk.ethereum.jsonrpc.serialization.{JsonEncoder, JsonMethodCodec, JsonMethodDecoder}
+import io.iohk.ethereum.transactions.TransactionHistoryService.ExtendedTransactionData
+import org.json4s.JsonAST._
+import org.json4s.Merge
+import JsonEncoder.OptionToNull._
+
+object MantisJsonMethodImplicits extends JsonMethodsImplicits {
+  implicit val extendedTransactionDataJsonEncoder: JsonEncoder[ExtendedTransactionData] = extendedTxData => {
+    val asTxResponse = TransactionResponse(
+      extendedTxData.stx,
+      extendedTxData.minedTransactionData.map(_.header),
+      extendedTxData.minedTransactionData.map(_.transactionIndex)
+    )
+
+    val encodedTxResponse = JsonEncoder.encode(asTxResponse)
+    val encodedExtension = JObject(
+      "isOutgoing" -> extendedTxData.isOutgoing.jsonEncoded,
+      "isPending" -> extendedTxData.isPending.jsonEncoded,
+      "gasUsed" -> extendedTxData.minedTransactionData.map(_.gasUsed).jsonEncoded,
+      "timestamp" -> extendedTxData.minedTransactionData.map(_.timestamp).jsonEncoded
+    )
+
+    Merge.merge(encodedTxResponse, encodedExtension)
+  }
+
+  implicit val mantis_getAccountTransactions
+      : JsonMethodCodec[GetAccountTransactionsRequest, GetAccountTransactionsResponse] =
+    new JsonMethodDecoder[GetAccountTransactionsRequest] with JsonEncoder[GetAccountTransactionsResponse] {
+      def decodeJson(params: Option[JArray]): Either[JsonRpcError, GetAccountTransactionsRequest] =
+        params match {
+          case Some(JArray(JString(addrJson) :: fromBlockJson :: toBlockJson :: Nil)) =>
+            for {
+              addr <- extractAddress(addrJson)
+              fromBlock <- extractQuantity(fromBlockJson)
+              toBlock <- extractQuantity(toBlockJson)
+            } yield GetAccountTransactionsRequest(addr, fromBlock to toBlock)
+          case _ => Left(InvalidParams())
+        }
+
+      override def encodeJson(t: GetAccountTransactionsResponse): JValue =
+        JObject("transactions" -> t.transactions.jsonEncoded)
+    }
+}

src/main/scala/io/iohk/ethereum/jsonrpc/MantisJsonMethodImplicits.scala

@@ -0,0 +1,35 @@
+package io.iohk.ethereum.jsonrpc
+import io.iohk.ethereum.domain.Address
+import io.iohk.ethereum.jsonrpc.MantisService.{GetAccountTransactionsRequest, GetAccountTransactionsResponse}
+import io.iohk.ethereum.jsonrpc.server.controllers.JsonRpcBaseController.JsonRpcConfig
+import io.iohk.ethereum.transactions.TransactionHistoryService
+import io.iohk.ethereum.transactions.TransactionHistoryService.ExtendedTransactionData
+import monix.eval.Task
+import cats.implicits._
+
+import scala.collection.immutable.NumericRange
+
+object MantisService {
+  case class GetAccountTransactionsRequest(address: Address, blocksRange: NumericRange[BigInt])
+  case class GetAccountTransactionsResponse(transactions: List[ExtendedTransactionData])
+}
+class MantisService(transactionHistoryService: TransactionHistoryService, jsonRpcConfig: JsonRpcConfig) {
+  def getAccountTransactions(
+      request: GetAccountTransactionsRequest
+  ): ServiceResponse[GetAccountTransactionsResponse] = {
+    if (request.blocksRange.length > jsonRpcConfig.accountTransactionsMaxBlocks) {
+      Task.now(
+        Left(
+          JsonRpcError.InvalidParams(
+            s"""Maximum number of blocks to search is ${jsonRpcConfig.accountTransactionsMaxBlocks}, requested: ${request.blocksRange.length}.
+               |See: 'mantis.network.rpc.account-transactions-max-blocks' config.""".stripMargin
+          )
+        )
+      )
+    } else {
+      transactionHistoryService
+        .getAccountTransactions(request.address, request.blocksRange)
+        .map(GetAccountTransactionsResponse(_).asRight)
+    }
+  }
+}