Merge pull request #667 from input-output-hk/etcm-107-backport-logging-configuration

[ETCM-107] Backport logging configuration

Author: kapke

build.sbt

@@ -31,24 +31,15 @@ val dep = {
     "ch.megard" %% "akka-http-cors" % "1.1.0",
     "org.json4s" %% "json4s-native" % "3.5.4",
     "de.heikoseeberger" %% "akka-http-json4s" % "1.34.0",
-    "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % "it,test",
     "io.suzaku" %% "boopickle" % "1.3.0",
     "org.ethereum" % "rocksdbjni" % rocksDb,
-    "org.scalatest" %% "scalatest" % "3.0.5" % "it,test",
-    "org.scalamock" %% "scalamock-scalatest-support" % "3.6.0" % "test",
-    "org.scalacheck" %% "scalacheck" % "1.14.0" % "it,test",
-    "ch.qos.logback" % "logback-classic" % "1.2.3",
-    "org.jline" % "jline" % "3.1.2",
-    "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.0",
     "io.circe" %% "circe-core" % circeVersion,
     "io.circe" %% "circe-generic" % circeVersion,
     "io.circe" %% "circe-parser" % circeVersion,
     "io.circe" %% "circe-generic-extras" % circeVersion,
-    "com.miguno.akka" %% "akka-mock-scheduler" % "0.5.5" % "it,test",
     "commons-io" % "commons-io" % "2.6",
     "org.scala-sbt.ipcsocket" % "ipcsocket" % "1.0.0",
     "org.bouncycastle" % "bcprov-jdk15on" % "1.59",
-    "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0",
     "org.typelevel" %% "mouse" % "0.23",
     "org.typelevel" %% "cats-core" % "2.0.0",
     "org.typelevel" %% "cats-effect" % "2.0.0",
@@ -61,12 +52,23 @@ val dep = {
     // mallet deps
     "org.jline" % "jline" % "3.1.2",
     "net.java.dev.jna" % "jna" % "4.5.1",
-    "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.5",
+    "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.0",
     "com.github.scopt" %% "scopt" % "3.7.0",
     // Metrics (https://github.com/DataDog/java-dogstatsd-client)
     "com.datadoghq" % "java-dogstatsd-client" % "2.5",
     "org.xerial.snappy" % "snappy-java" % "1.1.7.2",
-    "org.web3j" % "core" % "3.4.0" % "test"
+    // Logging
+    "ch.qos.logback" % "logback-classic" % "1.2.3",
+    "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0",
+    "net.logstash.logback" % "logstash-logback-encoder" % "6.4",
+    "org.codehaus.janino" % "janino" % "3.0.6",
+    // Test utils
+    "org.web3j" % "core" % "3.4.0" % "test",
+    "org.scalatest" %% "scalatest" % "3.0.5" % "it,test",
+    "com.miguno.akka" %% "akka-mock-scheduler" % "0.5.5" % "it,test",
+    "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % "it,test",
+    "org.scalamock" %% "scalamock-scalatest-support" % "3.6.0" % "test",
+    "org.scalacheck" %% "scalacheck" % "1.14.0" % "it,test"
   )
 }
 
@@ -146,6 +148,8 @@ mainClass in Compile := Some("io.iohk.ethereum.App")
 // Enables creating an executable with the configuration files, has to be run on the OS corresponding to the desired version
 jdkPackagerType := "image"
 
+Universal / mappings += (resourceDirectory in Compile).value / "logback.xml" -> "conf/logback.xml"
+
 val sep = java.io.File.separator
 jdkPackagerJVMArgs := Seq(
   "-Dconfig.file=." + sep + "conf" + sep + "mantis.conf",
@@ -155,7 +159,6 @@ jdkPackagerJVMArgs := Seq(
 
 coverageExcludedPackages := "io\\.iohk\\.ethereum\\.extvm\\.msg.*"
 
-
 addCommandAlias(
   "compile-all",
   """;compile

src/main/resources/application.conf

@@ -535,6 +535,17 @@ mantis {
     log-errors = true
   }
 
+  logging {
+    # Flag used to switch logs to the JSON format
+    json-output = false
+
+    # Logs directory
+    logs-dir = ${mantis.datadir}"/logs"
+
+    # Logs filename
+    logs-file = "mantis"
+  }
+
   async {
     dispatchers {
       block-forger {

src/main/resources/logback.xml

@@ -3,17 +3,33 @@
     <property name="stdoutEncoderPattern" value="%d{HH:mm:ss} [%logger{36}] - %msg%n" />
     <property name="fileEncoderPattern" value="%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} %X{akkaSource} - %msg%n" />
 
+    <!--read properties from application.conf-->
+    <newRule pattern="*/load" actionClass="io.iohk.ethereum.utils.LoadFromApplicationConfiguration"/>
+    <load key="mantis.logging.json-output" as="ASJSON"/>
+    <load key="mantis.logging.logs-dir" as="LOGSDIR"/>
+    <load key="mantis.logging.logs-file" as="LOGSFILENAME"/>
+
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>${stdoutEncoderPattern}</pattern>
         </encoder>
     </appender>
 
+    <appender name="STASH" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder class="net.logstash.logback.encoder.LogstashEncoder">
+            <customFields>{"hostname":"${HOSTNAME}"}</customFields>
+            <fieldNames>
+                <timestamp>timestamp</timestamp>
+                <version>[ignore]</version>
+            </fieldNames>
+        </encoder>
+    </appender>
+
     <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>${user.home}/.mantis/logs/mantis.log</file>
+        <file>${LOGSDIR}/${LOGSFILENAME}.log</file>
         <append>true</append>
         <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-            <fileNamePattern>${user.home}/.mantis/logs/mantis.%i.log.zip</fileNamePattern>
+            <fileNamePattern>${LOGSDIR}/${LOGSFILENAME}.%i.log.zip</fileNamePattern>
             <minIndex>1</minIndex>
             <maxIndex>10</maxIndex>
         </rollingPolicy>
@@ -26,7 +42,14 @@
     </appender>
 
     <root level="DEBUG">
-        <appender-ref ref="STDOUT" />
+        <if condition='p("ASJSON").contains("true")'>
+            <then>
+                <appender-ref ref="STASH" />
+            </then>
+            <else>
+                <appender-ref ref="STDOUT" />
+            </else>
+        </if>
         <appender-ref ref="FILE" />
     </root>
 

src/main/scala/io/iohk/ethereum/utils/LoadFromApplicationConfiguration.scala

@@ -0,0 +1,18 @@
+package io.iohk.ethereum.utils
+
+import ch.qos.logback.core.joran.action.Action
+import ch.qos.logback.core.joran.spi.InterpretationContext
+import com.typesafe.config.ConfigFactory
+import org.xml.sax.Attributes
+
+/**
+  * Make properties defined in application.conf available to logback
+  */
+class LoadFromApplicationConfiguration extends Action {
+
+  val config = ConfigFactory.load
+  override def begin(ic: InterpretationContext, body: String, attributes: Attributes): Unit = {
+    ic.addSubstitutionProperty(attributes.getValue("as"), config.getString(attributes.getValue("key")))
+  }
+  override def end(ic: InterpretationContext, body: String): Unit = ()
+}

src/main/scala/io/iohk/ethereum/utils/Logger.scala

@@ -1,8 +1,26 @@
 package io.iohk.ethereum.utils
 
+import com.typesafe.scalalogging
+import org.slf4j.{LoggerFactory, MDC}
+
 trait Logger {
-  import com.typesafe.scalalogging.Logger
+  val log: scalalogging.Logger = com.typesafe.scalalogging.Logger(LoggerFactory.getLogger(getClass))
+}
 
-  val log = Logger(getClass)
+trait LazyLogger {
+  lazy val log: scalalogging.Logger = com.typesafe.scalalogging.Logger(LoggerFactory.getLogger(getClass))
 }
 
+trait LoggingContext {
+  val asParameterMap: Map[String, String]
+}
+
+object LoggingContext {
+  implicit class ContextLoggerOps[T <: scalalogging.Logger](log: T) {
+    def withContext(context: LoggingContext)(doLog: T => Unit): Unit = {
+      context.asParameterMap.foreach { case (key, value) => MDC.put(key, value) }
+      doLog(log)
+      MDC.clear()
+    }
+  }
+}

src/universal/conf/logback.xml

@@ -0,0 +1,12 @@
+mantis {
+  logging {
+    # Flag used to switch logs to the JSON format
+    # json-output = false
+
+    # Logs directory
+    # logs-dir = ${mantis.datadir}"/logs"
+
+    # Logs filename
+    # logs-file = "mantis"
+  }
+}

src/universal/conf/logging.conf

@@ -15,6 +15,7 @@ include "sync.conf"
 include "misc.conf"
 include "consensus.conf"
 include "metrics.conf"
+include "logging.conf"
 
 # Uncomment to run Mantis in test-mode (similar to --test flag in cpp-ethereum).
 # When set, test validators and consensus are used by this node.