[ETCM-78 added checkpointing JRC

build.sbt

@@ -85,6 +85,7 @@ scalacOptions := Seq(
   "-Xlint:unsound-match",
   "-Ywarn-inaccessible",
   "-Ywarn-unused-import",
+  "-Ypartial-unification",
   "-encoding",
   "utf-8"
 )

insomnia_workspace.json

@@ -1,9 +1,95 @@
 {
   "_type": "export",
   "__export_format": 4,
-  "__export_date": "2020-09-30T20:01:40.792Z",
+  "__export_date": "2020-10-02T11:18:34.806Z",
   "__export_source": "insomnia.desktop.app:v2020.4.1",
   "resources": [
+    {
+      "_id": "req_4222a4d54ba24fa7813429bdcdb732df",
+      "parentId": "fld_a7212a5b96194230a7e0abc76ee2bf26",
+      "modified": 1601637263922,
+      "created": 1601637164399,
+      "url": "{{ node_url }}",
+      "name": "checkpointing_getLatestBlock",
+      "description": "",
+      "method": "POST",
+      "body": {
+        "mimeType": "application/json",
+        "text": "{\n\t\"jsonrpc\": \"2.0\",\n\t\"id\": 1,\n\t\"method\": \"checkpointing_getLatestBlock\",\n\t\"params\": [5]\n}"
+      },
+      "parameters": [],
+      "headers": [
+        {
+          "name": "Content-Type",
+          "value": "application/json",
+          "id": "pair_a28efff3b44442d99b7f3fbc54d8c9b6"
+        }
+      ],
+      "authentication": {},
+      "metaSortKey": -1601637164399,
+      "isPrivate": false,
+      "settingStoreCookies": true,
+      "settingSendCookies": true,
+      "settingDisableRenderRequestBody": false,
+      "settingEncodeUrl": true,
+      "settingRebuildPath": true,
+      "settingFollowRedirects": "global",
+      "_type": "request"
+    },
+    {
+      "_id": "fld_a7212a5b96194230a7e0abc76ee2bf26",
+      "parentId": "wrk_097d43914a4d4aea8b6f73f647921182",
+      "modified": 1601637156313,
+      "created": 1601637156313,
+      "name": "Checkpointing",
+      "description": "",
+      "environment": {},
+      "environmentPropertyOrder": null,
+      "metaSortKey": -1601637156313,
+      "_type": "request_group"
+    },
+    {
+      "_id": "wrk_097d43914a4d4aea8b6f73f647921182",
+      "parentId": null,
+      "modified": 1599825617921,
+      "created": 1552662762769,
+      "name": "Mantis",
+      "description": "",
+      "scope": null,
+      "_type": "workspace"
+    },
+    {
+      "_id": "req_da1e409360394849b673ec4e27f542b6",
+      "parentId": "fld_a7212a5b96194230a7e0abc76ee2bf26",
+      "modified": 1601637193229,
+      "created": 1601637186866,
+      "url": "{{ node_url }}",
+      "name": "checkpointing_pushCheckpoint",
+      "description": "",
+      "method": "POST",
+      "body": {
+        "mimeType": "application/json",
+        "text": "{\n\t\"jsonrpc\": \"2.0\",\n\t\"id\": 1,\n\t\"method\": \"checkpointing_pushCheckpoint\",\n\t\"params\": [\n\t\t\"127d6fde40d20208641c057a1ad4d12d44433881a660b15ac99f04f25762fb9b\",\n\t\t[\n\t\"2194b40851c648e7570e75ea2c507887d11c2270f7523469953fc5c3d5e0f50f48d73ea0b827eb81bb2fc0511f09d10b8f1d3f88e251ed231bb0f5cd03826d281b\",\n\t\t\t\"bbd4ae567202a6e7f40826c964a918760253596bb92052ea7ef4b30338b19fc12d56d497c88f0f13eff0ad542a8a4c1069559cb43e9741b849bf6577287450e31b\"\n\t\t]\n\t]\n}"
+      },
+      "parameters": [],
+      "headers": [
+        {
+          "name": "Content-Type",
+          "value": "application/json",
+          "id": "pair_a28efff3b44442d99b7f3fbc54d8c9b6"
+        }
+      ],
+      "authentication": {},
+      "metaSortKey": -1583661254601.5,
+      "isPrivate": false,
+      "settingStoreCookies": true,
+      "settingSendCookies": true,
+      "settingDisableRenderRequestBody": false,
+      "settingEncodeUrl": true,
+      "settingRebuildPath": true,
+      "settingFollowRedirects": "global",
+      "_type": "request"
+    },
     {
       "_id": "req_cd0078ce4a034ebdbdf7dc9e20e78a29",
       "parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
@@ -53,16 +139,6 @@
       "metaSortKey": -1600249374160,
       "_type": "request_group"
     },
-    {
-      "_id": "wrk_097d43914a4d4aea8b6f73f647921182",
-      "parentId": null,
-      "modified": 1599825617921,
-      "created": 1552662762769,
-      "name": "Mantis",
-      "description": "",
-      "scope": null,
-      "_type": "workspace"
-    },
     {
       "_id": "req_6197fefa1e1448a89f30712ec12295f8",
       "parentId": "fld_2b54cbb84e244284b3ef752c5f805376",
@@ -1034,12 +1110,18 @@
       "modified": 1599825641645,
       "created": 1552663140073,
       "name": "Develop",
-      "data": { "node_url": "http://127.0.0.1:8546" },
-      "dataPropertyOrder": { "&": ["node_url"] },
+      "data": {
+        "node_url": "http://127.0.0.1:8546"
+      },
+      "dataPropertyOrder": {
+        "&": [
+          "node_url"
+        ]
+      },
       "color": null,
       "isPrivate": false,
       "metaSortKey": 1552663140073,
       "_type": "environment"
     }
   ]
-}
+}

src/main/resources/application.conf

@@ -185,8 +185,8 @@ mantis {
       }
 
       # Enabled JSON-RPC APIs over the JSON-RPC endpoint
-      # Available choices are: eth, web3, net, personal, test, daedalus, iele, qa
-      apis = "eth,web3,net,personal,daedalus,debug,qa"
+      # Available choices are: web3, eth, net, personal, daedalus, test, iele, debug, qa, checkpointing
+      apis = "eth,web3,net,personal,daedalus,debug,qa,checkpointing"
 
       # Maximum number of blocks for daedalus_getAccountTransactions
       account-transactions-max-blocks = 50000

src/main/scala/io/iohk/ethereum/blockchain/sync/regular/RegularSync.scala

@@ -1,7 +1,9 @@
 package io.iohk.ethereum.blockchain.sync.regular
 
 import akka.actor.{Actor, ActorLogging, ActorRef, AllForOneStrategy, Cancellable, Props, Scheduler, SupervisorStrategy}
+import akka.util.ByteString
 import io.iohk.ethereum.blockchain.sync.BlockBroadcast
+import io.iohk.ethereum.crypto.ECDSASignature
 import io.iohk.ethereum.domain.{Block, Blockchain}
 import io.iohk.ethereum.ledger.Ledger
 import io.iohk.ethereum.utils.Config.SyncConfig
@@ -95,4 +97,5 @@ object RegularSync {
   sealed trait RegularSyncMsg
   case object Start extends RegularSyncMsg
   case class MinedBlock(block: Block) extends RegularSyncMsg
+  case class NewCheckpoint(parentHash: ByteString, signatures: Seq[ECDSASignature]) extends RegularSyncMsg
 }

src/main/scala/io/iohk/ethereum/jsonrpc/CheckpointingJsonMethodsImplicits.scala

@@ -0,0 +1,64 @@
+package io.iohk.ethereum.jsonrpc
+
+import akka.util.ByteString
+import io.iohk.ethereum.crypto.ECDSASignature
+import io.iohk.ethereum.jsonrpc.CheckpointingService._
+import io.iohk.ethereum.jsonrpc.JsonRpcController.Codec
+import io.iohk.ethereum.jsonrpc.JsonRpcErrors.InvalidParams
+import io.iohk.ethereum.jsonrpc.JsonSerializers.QuantitiesSerializer
+import org.json4s.JsonAST._
+import org.json4s.{Extraction, JsonAST}
+
+object CheckpointingJsonMethodsImplicits extends JsonMethodsImplicits {
+
+  implicit val checkpointing_getLatestBlock: Codec[GetLatestBlockRequest, GetLatestBlockResponse] =
+    new Codec[GetLatestBlockRequest, GetLatestBlockResponse] {
+      override def decodeJson(
+        params: Option[JsonAST.JArray]
+      ): Either[JsonRpcError, GetLatestBlockRequest] =
+        params match {
+          case Some(JArray(JInt(chkpInterval) :: Nil)) =>
+            if (chkpInterval > 0 && chkpInterval <= Int.MaxValue)
+              Right(GetLatestBlockRequest(chkpInterval.toInt))
+            else
+              Left(InvalidParams("Expected positive integer"))
+          case _ =>
+            Left(InvalidParams())
+        }
+
+      override def encodeJson(resp: GetLatestBlockResponse): JsonAST.JValue =
+        Extraction.decompose(resp)(formats - QuantitiesSerializer)
+    }
+
+  implicit val checkpointing_pushCheckpoint: Codec[PushCheckpointRequest, PushCheckpointResponse] =
+    new Codec[PushCheckpointRequest, PushCheckpointResponse] {
+      override def decodeJson(
+        params: Option[JsonAST.JArray]
+      ): Either[JsonRpcError, PushCheckpointRequest] =
+        params match {
+          case Some(JArray(JString(hashStr) :: (signatures: JArray) :: Nil)) =>
+            for {
+              hash <- extractHash(hashStr)
+              sigs <- extractSignatures(signatures)
+            } yield PushCheckpointRequest(hash, sigs)
+
+          case _ =>
+            Left(InvalidParams())
+        }
+
+      override def encodeJson(t: PushCheckpointResponse): JsonAST.JValue =
+        JBool(true)
+    }
+
+  private def extractSignatures(arr: JArray): Either[JsonRpcError, List[ECDSASignature]] = {
+    import cats.implicits._
+    def parseSig(bs: ByteString) =
+      ECDSASignature.fromBytes(bs).toRight(InvalidParams("Bad signature length"))
+
+    arr.arr.traverse {
+      case JString(str) => extractBytes(str).flatMap(parseSig)
+
+      case other => Left(InvalidParams(s"Unable to extract a signature from: $other"))
+    }
+  }
+}

src/main/scala/io/iohk/ethereum/jsonrpc/CheckpointingService.scala

@@ -0,0 +1,52 @@
+package io.iohk.ethereum.jsonrpc
+
+import akka.actor.ActorRef
+import akka.util.ByteString
+import io.iohk.ethereum.blockchain.sync.regular.RegularSync.NewCheckpoint
+import io.iohk.ethereum.crypto.ECDSASignature
+import io.iohk.ethereum.domain.Blockchain
+import io.iohk.ethereum.utils.Logger
+import monix.execution.Scheduler.Implicits.global
+
+import scala.concurrent.Future
+
+class CheckpointingService(
+  blockchain: Blockchain,
+  syncController: ActorRef
+) extends Logger {
+
+  import CheckpointingService._
+
+  def getLatestBlock(req: GetLatestBlockRequest): ServiceResponse[GetLatestBlockResponse] = {
+    lazy val bestBlockNum = blockchain.getBestBlockNumber()
+    lazy val blockToReturnNum = bestBlockNum - bestBlockNum % req.checkpointingInterval
+
+    Future {
+      blockchain.getBlockByNumber(blockToReturnNum)
+    }.flatMap {
+      case Some(b) =>
+        val resp = GetLatestBlockResponse(b.hash, b.number)
+        Future.successful(Right(resp))
+
+      case None =>
+        log.error(
+          s"Failed to retrieve block for checkpointing: block at number $blockToReturnNum was unavailable " +
+            s"even though best block number was $bestBlockNum (re-org occurred?)"
+        )
+        getLatestBlock(req) // this can fail only during a re-org, so we just try again
+    }
+  }
+
+  def pushCheckpoint(req: PushCheckpointRequest): ServiceResponse[PushCheckpointResponse] = Future {
+    syncController ! NewCheckpoint(req.hash, req.signatures)
+    Right(PushCheckpointResponse())
+  }
+}
+
+object CheckpointingService {
+  case class GetLatestBlockRequest(checkpointingInterval: Int)
+  case class GetLatestBlockResponse(hash: ByteString, number: BigInt)
+
+  case class PushCheckpointRequest(hash: ByteString, signatures: List[ECDSASignature])
+  case class PushCheckpointResponse()
+}

src/main/scala/io/iohk/ethereum/jsonrpc/JsonMethodsImplicits.scala

@@ -9,12 +9,7 @@ import io.iohk.ethereum.jsonrpc.EthService.BlockParam
 import io.iohk.ethereum.jsonrpc.JsonRpcController.JsonDecoder.NoParamsDecoder
 import io.iohk.ethereum.jsonrpc.JsonRpcController.{Codec, JsonDecoder, JsonEncoder}
 import io.iohk.ethereum.jsonrpc.JsonRpcErrors.InvalidParams
-import io.iohk.ethereum.jsonrpc.JsonSerializers.{
-  AddressJsonSerializer,
-  OptionNoneToJNullSerializer,
-  QuantitiesSerializer,
-  UnformattedDataJsonSerializer
-}
+import io.iohk.ethereum.jsonrpc.JsonSerializers.{AddressJsonSerializer, OptionNoneToJNullSerializer, QuantitiesSerializer, UnformattedDataJsonSerializer}
 import io.iohk.ethereum.jsonrpc.NetService._
 import io.iohk.ethereum.jsonrpc.PersonalService._
 import io.iohk.ethereum.jsonrpc.Web3Service.{ClientVersionRequest, ClientVersionResponse, Sha3Request, Sha3Response}

src/main/scala/io/iohk/ethereum/jsonrpc/JsonRpcController.scala

@@ -16,6 +16,9 @@ import io.iohk.ethereum.jsonrpc.TestService._
 import io.iohk.ethereum.jsonrpc.server.http.JsonRpcHttpServer.JsonRpcHttpServerConfig
 import io.iohk.ethereum.jsonrpc.server.ipc.JsonRpcIpcServer.JsonRpcIpcServerConfig
 import java.util.concurrent.TimeUnit
+
+import io.iohk.ethereum.jsonrpc.CheckpointingService._
+
 import scala.concurrent.Future
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.FiniteDuration
@@ -69,7 +72,7 @@ object JsonRpcController {
         override val apis: Seq[String] = {
           val providedApis = rpcConfig.getString("apis").split(",").map(_.trim.toLowerCase)
           val invalidApis =
-            providedApis.diff(List("web3", "eth", "net", "personal", "daedalus", "test", "iele", "debug", "qa"))
+            providedApis.diff(Apis.available)
           require(invalidApis.isEmpty, s"Invalid RPC APIs specified: ${invalidApis.mkString(",")}")
           providedApis
         }
@@ -87,15 +90,16 @@ object JsonRpcController {
     val Eth = "eth"
     val Web3 = "web3"
     val Net = "net"
-    val Db = "db"
     val Personal = "personal"
     val Daedalus = "daedalus"
-    val Admin = "admin"
     val Debug = "debug"
     val Rpc = "rpc"
     val Test = "test"
     val Iele = "iele"
     val Qa = "qa"
+    val Checkpointing = "checkpointing"
+
+    val available = Seq(Eth, Web3, Net, Personal, Daedalus, Debug, Test, Iele, Qa, Checkpointing)
   }
 
 }
@@ -108,6 +112,7 @@ class JsonRpcController(
     testServiceOpt: Option[TestService],
     debugService: DebugService,
     qaService: QAService,
+    checkpointingService: CheckpointingService,
     config: JsonRpcConfig
 ) extends Logger {
 
@@ -119,20 +124,20 @@ class JsonRpcController(
   import JsonRpcErrors._
   import DebugJsonMethodsImplicits._
   import QAJsonMethodsImplicits._
+  import CheckpointingJsonMethodsImplicits._
 
   lazy val apisHandleFns: Map[String, PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]]] = Map(
     Apis.Eth -> handleEthRequest,
     Apis.Web3 -> handleWeb3Request,
     Apis.Net -> handleNetRequest,
-    Apis.Db -> PartialFunction.empty,
     Apis.Personal -> handlePersonalRequest,
     Apis.Daedalus -> handleDaedalusRequest,
     Apis.Rpc -> handleRpcRequest,
-    Apis.Admin -> PartialFunction.empty,
     Apis.Debug -> handleDebugRequest,
     Apis.Test -> handleTestRequest,
     Apis.Iele -> handleIeleRequest,
-    Apis.Qa -> handleQARequest
+    Apis.Qa -> handleQARequest,
+    Apis.Checkpointing -> handleCheckpointingRequest
   )
 
   private def enabledApis: Seq[String] = config.apis :+ Apis.Rpc // RPC enabled by default
@@ -349,6 +354,14 @@ class JsonRpcController(
       handle[GetPendingTransactionsRequest, GetPendingTransactionsResponse](qaService.getPendingTransactions, req)
   }
 
+  private def handleCheckpointingRequest: PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]] = {
+    case req @ JsonRpcRequest(_, "checkpointing_getLatestBlock", _, _) =>
+      handle[GetLatestBlockRequest, GetLatestBlockResponse](checkpointingService.getLatestBlock, req)
+
+    case req @ JsonRpcRequest(_, "checkpointing_pushCheckpoint", _, _) =>
+      handle[PushCheckpointRequest, PushCheckpointResponse](checkpointingService.pushCheckpoint, req)
+  }
+
   private def handleRpcRequest: PartialFunction[JsonRpcRequest, Future[JsonRpcResponse]] = {
     case req @ JsonRpcRequest(_, "rpc_modules", _, _) =>
       val result = enabledApis.map { _ -> "1.0" }.toMap