[ETCM-129] Scala 2.13 (#875)

* [ETCM-129] - Migrate to 2.13

Co-authored-by: dmitry-worker <[email protected]>
Co-authored-by: Jonathan Ringer <[email protected]>
Guided by: Piotr Paradzinski <[email protected]>
Reviewed by: Enrique Rodríguez <[email protected]>
Dependencies upgrade guide: Akosh Farkash <[email protected]>

Author: 3 people

.buildkite/pipeline.nix

@@ -14,24 +14,7 @@ in
     nixExpr = commonAttrs // {
       label = "ensure Nix expressions are up-to-date";
       command = ''
-        echo "Checking if Nix expressions are up-to-date..."
-
-        nix-shell --run 'sbtix-gen-all2'
-
-        set +e
-        git diff --exit-code > nix-expr.patch
-        if [ "$?" -eq "1" ]; then
-          set -e
-          echo "Nix expressions not up-to-date."
-          echo "Download and apply the patch available in the artifact paths of this step:"
-          echo "  patch -p1 < nix-expr.patch"
-          echo "Aborting."
-          exit 1
-        else
-          set -e
-          echo "Nix expressions up-to-date!"
-          exit 0
-        fi
+        ./update-nix.sh --check
       '';
       retry.automatic = false;
       artifactPaths = [
@@ -70,8 +53,8 @@ in
       '';
       artifactPaths = [
         "bytes/target/test-reports/**/*"
-        "bytes/target/scala-2.12/scoverage-report/**/*"
-        "bytes/target/scala-2.12/coverage-report/**/*"
+        "bytes/target/scala-2.13/scoverage-report/**/*"
+        "bytes/target/scala-2.13/coverage-report/**/*"
       ];
     };
 
@@ -83,8 +66,8 @@ in
       '';
       artifactPaths = [
         "crypto/target/test-reports/**/*"
-        "crypto/target/scala-2.12/scoverage-report/**/*"
-        "crypto/target/scala-2.12/coverage-report/**/*"
+        "crypto/target/scala-2.13/scoverage-report/**/*"
+        "crypto/target/scala-2.13/coverage-report/**/*"
       ];
     };
 
@@ -96,8 +79,8 @@ in
       '';
       artifactPaths = [
         "rlp/target/test-reports/**/*"
-        "rlp/target/scala-2.12/scoverage-report/**/*"
-        "rlp/target/scala-2.12/coverage-report/**/*"
+        "rlp/target/scala-2.13/scoverage-report/**/*"
+        "rlp/target/scala-2.13/coverage-report/**/*"
       ];
     };
 
@@ -109,8 +92,8 @@ in
       '';
       artifactPaths = [
         "target/test-reports/**/*"
-        "target/scala-2.12/scoverage-report/**/*"
-        "target/scala-2.12/coverage-report/**/*"
+        "target/scala-2.13/scoverage-report/**/*"
+        "target/scala-2.13/coverage-report/**/*"
       ];
     };
 
@@ -122,8 +105,8 @@ in
       '';
       artifactPaths = [
         "target/test-reports/**/*"
-        "target/scala-2.12/scoverage-report/**/*"
-        "target/scala-2.12/coverage-report/**/*"
+        "target/scala-2.13/scoverage-report/**/*"
+        "target/scala-2.13/coverage-report/**/*"
       ];
     };
 

.buildkite/shell.nix

@@ -17,17 +17,12 @@ let
 
     exec ${pkgs.protobuf}/bin/protoc "$@"
   '';
-
-  sbtix = pkgs.callPackage sources.Sbtix { };
-
 in
 
   with pkgs;
-  
+
   mkShell {
-  
-    buildInputs = [ sbt solc jdk8 protoc-wrapper sbtix ];
+    nativeBuildInputs = [ sbt solc jdk8 protoc-wrapper ];
     # SBT = "sbt -v -mem 2048 -J-Xmx4g -Dsbt.ivy.home=/cache/ivy2 -Dsbt.boot.directory=/cache/sbt -Dmaven.repo.local=/cache/maven -Dnix=true";
     SBT = "sbt -v -mem 2048 -J-Xmx4g -Dnix=true";
-    SBTIX_GEN = "true";
   }

.gitignore

@@ -22,3 +22,4 @@ metals.sbt
 
 # Nix
 result
+.nix/

README.md

@@ -116,40 +116,20 @@ In the root of the project:
 nix-build
 ```
 
-##### Regenerate lock files
+##### Update sbt+nix dependencies
 
-```
-nix-shell
-sbtix-gen-all2
-```
-
-OR
-
-If the "ensure Nix expressions are up-to-date" step of your CI
-build has failed, check the artifacts of that step. There should be a
-patch provided, which you can apply locally with:
+When updating project dependencies, the nix fixed-output-derivation
+will need to be updated so that it includes the new dependency state.
 
+To do so, please run:
 ```
-patch -p1 < downloaded.patch
+./update-nix.sh
+git add ./nix/pkgs/mantis.nix
+git commit -m "Update nix-sbt sha"
 ```
 
-This patch will update the lock files for you.
-
-###### Why so many lock files?
-
-- `repo.nix`                 : generated by the `sbtix-gen` command and includes only the build dependencies for the project.
-- `project/repo.nix`         : generated by the `sbtix-gen-all` command and includes only the plugin dependencies. Also generates `repo.nix`.
-- `project/project/repo.nix` : generated by the `sbtix-gen-all2` command and includes only the plugin dependencies. Also generates `repo.nix` and `project/repo.nix`.
-
-##### error: unsupported argument 'submodules' to 'fetchGit'
-
-You get this error when you aren't using a new-enough version of Nix (fetchGit support for submodules is recent).
-
-To fix this, update the version of Nix you are using, or in a pinch:
-
-  - Remove the "submodules = true;" argument from fetchGit (in `./nix/pkgs/mantis/default.nix`).
-  - `git submodule update --recursive --init`
-  - `nix-build`
+*NOTE:* This should only be necessary when updating dependencies
+(For example, edits to build.sbt or project/plugins.sbt will likely need to be regenerated)
 
 ### Monitoring
 

build.sbt

@@ -16,7 +16,7 @@ def commonSettings(projectName: String): Seq[sbt.Def.Setting[_]] = Seq(
   name := projectName,
   organization := "io.iohk",
   version := "3.2.1",
-  scalaVersion := "2.12.12",
+  scalaVersion := "2.13.4",
   // Scalanet snapshots are published to Sonatype after each build.
   resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
   testOptions in Test += Tests
@@ -26,10 +26,6 @@ def commonSettings(projectName: String): Seq[sbt.Def.Setting[_]] = Seq(
     "-deprecation",
     "-feature",
     "-Xfatal-warnings",
-    "-Xlint:unsound-match",
-    "-Ywarn-inaccessible",
-    "-Ywarn-unused-import",
-    "-Ypartial-unification",
     "-encoding",
     "utf-8"
   ),
@@ -211,7 +207,9 @@ lazy val node = {
   if (!nixBuild)
     node
   else
+    //node.settings(PB.protocExecutable := file("protoc"))
     node.settings(PB.runProtoc in Compile := (args => Process("protoc", args) !))
+
 }
 
 coverageExcludedPackages := "io\\.iohk\\.ethereum\\.extvm\\.msg.*"

bytes/src/main/scala/io/iohk/ethereum/utils/ByteStringUtils.scala

@@ -2,11 +2,68 @@ package io.iohk.ethereum.utils
 
 import akka.util.ByteString
 
+import scala.collection.mutable
+import scala.math.Ordering.Implicits._
+
 object ByteStringUtils {
   def hash2string(hash: ByteString): String =
     Hex.toHexString(hash.toArray[Byte])
 
   // unsafe
   def string2hash(hash: String): ByteString =
     ByteString(Hex.decode(hash))
+
+
+  implicit class Padding(val bs: ByteString) extends AnyVal {
+    def padToByteString(length: Int, b: Byte): ByteString = {
+      if (length <= bs.length) bs else {
+        val len = Math.max(bs.length, length)
+        val result = new Array[Byte](len)
+        bs.copyToArray(result, 0)
+        var i = bs.length
+        while (i < len) {
+          result.update(i, b)
+          i += 1
+        }
+        ByteString.fromArray(result)
+      }
+    }
+  }
+
+
+  sealed trait ByteStringElement {
+    def len: Int
+    def asByteArray: Array[Byte]
+  }
+
+  implicit class ByteStringSelfElement(val bs: ByteString) extends ByteStringElement {
+    def len: Int = bs.length
+    def asByteArray: Array[Byte] = bs.toArray
+  }
+
+  implicit class ByteStringArrayElement(val ar: Array[Byte]) extends ByteStringElement {
+    def len: Int = ar.length
+    def asByteArray: Array[Byte] = ar
+  }
+
+  implicit class ByteStringByteElement(val b: Byte) extends ByteStringElement {
+    def len: Int = 1
+    def asByteArray: Array[Byte] = Array(b)
+  }
+
+  implicit val byteStringOrdering: Ordering[ByteString] = {
+    Ordering.by[ByteString, Seq[Byte]](_.toSeq)
+  }
+
+  def concatByteStrings(head: ByteStringElement, tail: ByteStringElement*): ByteString = {
+    val it = Iterator.single(head) ++ tail.iterator
+    concatByteStrings(it)
+  }
+
+  def concatByteStrings(elements: Iterator[ByteStringElement]): ByteString = {
+    val builder = new mutable.ArrayBuilder.ofByte
+    elements.foreach(el => builder.addAll(el.asByteArray))
+    ByteString(builder.result())
+  }
+
 }

bytes/src/main/scala/io/iohk/ethereum/utils/ByteUtils.scala

@@ -97,8 +97,8 @@ object ByteUtils {
 
   def padLeft(bytes: ByteString, length: Int, byte: Byte = 0): ByteString = {
     val l = math.max(0, length - bytes.length)
-    val fill = Seq.fill[Byte](l)(byte)
-    fill ++: bytes
+    val fill = Array.fill[Byte](l)(byte)
+    ByteString.apply(fill) ++ bytes
   }
 
   def compactPickledBytesToArray(buffer: ByteBuffer): Array[Byte] = {
@@ -141,7 +141,7 @@ object ByteUtils {
     * @param bigEndian - param specifying which int representation should be used.
     * @return Unit
     */
-  def intsToBytesMut(arr: Array[Int], b: Array[Byte], bigEndian: Boolean) {
+  def intsToBytesMut(arr: Array[Int], b: Array[Byte], bigEndian: Boolean): Unit = {
     if (!bigEndian) {
       var off = 0
       var i = 0
@@ -187,7 +187,7 @@ object ByteUtils {
     * @param bigEndian - param specifying which int representation should be used.
     * @return Unit
     */
-  def bytesToIntsMut(b: Array[Byte], arr: Array[Int], bigEndian: Boolean) {
+  def bytesToIntsMut(b: Array[Byte], arr: Array[Int], bigEndian: Boolean): Unit = {
     if (!bigEndian) {
       var off = 0
       var i = 0

bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala

@@ -5,5 +5,5 @@ object Hex {
     bytes.map("%02x".format(_)).mkString
 
   def decode(hex: String): Array[Byte] =
-    hex.replaceAll("[^0-9A-Fa-f]", "").sliding(2, 2).toArray.map(Integer.parseInt(_, 16).toByte)
+    hex.toSeq.sliding(2, 2).toArray.map(s=> Integer.parseInt(s.unwrap, 16).toByte)
 }

bytes/src/test/scala/io/iohk/ethereum/utils/ByteStringUtilsTest.scala

@@ -0,0 +1,61 @@
+package io.iohk.ethereum.utils
+
+import akka.util.ByteString
+import org.scalatest.wordspec.AnyWordSpec
+import org.scalatest.matchers.should.Matchers
+import ByteStringUtils._
+import scala.collection.immutable.ArraySeq
+import scala.util.{Try, Success, Failure}
+
+class ByteStringUtilsTest extends AnyWordSpec with Matchers {
+
+  "ByteStringUtilsTest" should {
+
+    "succeed parsing a valid hash string" in {
+      val validHashString = "00FF00FF"
+      val parsed = Try(string2hash(validHashString))
+      val expected = ByteString(Array[Byte](0, -1, 0, -1))
+      parsed shouldEqual Success(expected)
+    }
+
+    "fail parsing a valid hash string" in {
+      val invalidHashString = "XXYYZZXX"
+      val parsed = Try(string2hash(invalidHashString))
+      parsed shouldBe a[Failure[_]]
+    }
+
+    "concatByteStrings for simple bytestrings" in {
+      val bs1 = string2hash("0000")
+      val bs2 = string2hash("FFFF")
+      val summarized: ByteString = bs1 ++ bs2
+
+      val concatenated: ByteString = ByteStringUtils.concatByteStrings(bs1, bs2)
+      summarized shouldEqual concatenated
+    }
+
+    "concatByteStrings for various argument types" in {
+      val bs1 = string2hash("0000")
+      val bs2 = string2hash("FFFF")
+      val bs3: Byte = 2
+      val bs4 = Array[Byte](3, 3)
+      val bs5 = Array[Byte](4, 4)
+      val summarized: ByteString = bs1 ++ bs2
+      val concatenated: ByteString = ByteStringUtils.concatByteStrings(bs1, bs2, bs3, bs4, bs5)
+      concatenated shouldEqual string2hash("0000FFFF0203030404")
+    }
+
+    "apply padding the same way seqOps does" in {
+      val bsu = string2hash("0000FFFF")
+      val seq = ArraySeq.unsafeWrapArray(bsu.toArray)
+      bsu.padToByteString(3, 0) shouldEqual bsu // result is ByteString
+      bsu.padTo(3,0) shouldEqual seq            // result is Seq
+
+      val longSeq = ArraySeq[Byte](0, 0, -1, -1, 1 ,1)
+      val longBsu = string2hash("0000FFFF0101")
+      bsu.padToByteString(6, 1) shouldEqual longBsu
+      bsu.padTo(6, 1) shouldEqual longSeq
+    }
+
+  }
+}
+

crypto/src/main/scala/io/iohk/ethereum/crypto/ECDSASignature.scala

@@ -64,6 +64,7 @@ object ECDSASignature {
       case Some(id) if v == negativePointSign => (id * 2 + newNegativePointSign).toByte
       case Some(id) if v == positivePointSign => (id * 2 + newPositivePointSign).toByte
       case None => v
+      case _    => throw new IllegalStateException(s"Unexpected pointSign. ChainId: ${chainId}, v: ${v}")
     }
 
     ECDSASignature(r, s, pointSign)
@@ -176,6 +177,7 @@ case class ECDSASignature(r: BigInt, s: BigInt, v: Byte) {
     def bigInt2Bytes(b: BigInt) =
       ByteUtils.padLeft(ByteString(b.toByteArray).takeRight(RLength), RLength, 0)
 
-    bigInt2Bytes(r) ++ bigInt2Bytes(s) :+ v
+
+    bigInt2Bytes(r) ++ bigInt2Bytes(s) ++ ByteString(v)
   }
 }