MapR [SPARK-901] Spark-3.1.1 doesn't start by warden (apache#840)

Co-authored-by: Egor Krivokon <>

Author: ekrivokonmapr

build/dev-build.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-PROFILES="-Pyarn -Phadoop-provided -Pscala-2.11 -Phive -Phive-thriftserver -DskipTests"
+PROFILES="-Pyarn -Phadoop-provided -Pscala-2.12 -Phive -Phive-thriftserver -Dskip-kafka-0-8 -DskipTests"
 
 if [[ -z "$1" ]]; then
     ./build/mvn ${PROFILES} clean install
@@ -12,5 +12,5 @@ fi
 
 if [ $? -ne 0 ]; then exit 1; fi
 
-scp -r assembly/target/scala-2.11/jars mapr@node1:/opt/mapr/spark/spark-2.4.5/jars
+scp -r assembly/target/scala-2.11/jars mapr@node1:/opt/mapr/spark/spark-3.1.1/jars
 if [ $? -ne 0 ]; then exit 1; fi

ext-conf/warden.spark-historyserver.conf

@@ -1,12 +1,12 @@
 services=spark-historyserver:1:cldb
 service.displayname=SparkHistoryServer
-service.command.start=/opt/mapr/spark/spark-2.4.5/sbin/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1
-service.command.stop=/opt/mapr/spark/spark-2.4.5/sbin/spark-daemon.sh stop org.apache.spark.deploy.history.HistoryServer 1
-service.command.monitorcommand=/opt/mapr/spark/spark-2.4.5/sbin/spark-daemon.sh status org.apache.spark.deploy.history.HistoryServer 1
+service.command.start=/opt/mapr/spark/spark-3.1.1/sbin/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1
+service.command.stop=/opt/mapr/spark/spark-3.1.1/sbin/spark-daemon.sh stop org.apache.spark.deploy.history.HistoryServer 1
+service.command.monitorcommand=/opt/mapr/spark/spark-3.1.1/sbin/spark-daemon.sh status org.apache.spark.deploy.history.HistoryServer 1
 service.command.type=BACKGROUND
 service.ui.port=18080
 service.uri=/
-service.logs.location=/opt/mapr/spark/spark-2.4.5/logs/
+service.logs.location=/opt/mapr/spark/spark-3.1.1/logs/
 service.process.type=JAVA
 service.alarm.tersename=nasparkh
 service.alarm.label=SPARKHISTORYDOWNALARM

ext-conf/warden.spark-master.conf

@@ -1,13 +1,13 @@
 services=spark-master:all:cldb
 service.displayname=SparkMaster
-service.command.start=/opt/mapr/spark/spark-2.4.5/sbin/spark-daemon.sh start org.apache.spark.deploy.master.Master 1
-service.command.stop=/opt/mapr/spark/spark-2.4.5/sbin/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1
-service.command.monitorcommand=/opt/mapr/spark/spark-2.4.5/sbin/spark-daemon.sh status org.apache.spark.deploy.master.Master 1
+service.command.start=/opt/mapr/spark/spark-3.1.1/sbin/spark-daemon.sh start org.apache.spark.deploy.master.Master 1
+service.command.stop=/opt/mapr/spark/spark-3.1.1/sbin/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1
+service.command.monitorcommand=/opt/mapr/spark/spark-3.1.1/sbin/spark-daemon.sh status org.apache.spark.deploy.master.Master 1
 service.command.type=BACKGROUND
 service.port=7077
 service.ui.port=8580
 service.uri=/
-service.logs.location=/opt/mapr/spark/spark-2.4.5/logs/
+service.logs.location=/opt/mapr/spark/spark-3.1.1/logs/
 service.process.type=JAVA
 service.alarm.tersename=naspark
 service.alarm.label=SPARKDOWNALARM

ext-conf/warden.spark-thriftserver.conf

@@ -1,13 +1,13 @@
 services=spark-thriftserver:1:cldb
 service.displayname=SparkThriftServer
-service.command.start=/opt/mapr/spark/spark-2.4.5/sbin/start-thriftserver.sh --conf spark.ssl.keyStore=/opt/mapr/conf/ssl_keystore
-service.command.stop=/opt/mapr/spark/spark-2.4.5/sbin/stop-thriftserver.sh
-service.command.monitorcommand=/opt/mapr/spark/spark-2.4.5/sbin/spark-daemon.sh status org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1
+service.command.start=/opt/mapr/spark/spark-3.1.1/sbin/start-thriftserver.sh --conf spark.ssl.keyStore=/opt/mapr/conf/ssl_keystore
+service.command.stop=/opt/mapr/spark/spark-3.1.1/sbin/stop-thriftserver.sh
+service.command.monitorcommand=/opt/mapr/spark/spark-3.1.1/sbin/spark-daemon.sh status org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1
 service.command.type=BACKGROUND
 service.port=2304
 service.ui.port=4040
 service.uri=/
-service.logs.location=/opt/mapr/spark/spark-2.4.5/logs/
+service.logs.location=/opt/mapr/spark/spark-3.1.1/logs/
 service.process.type=JAVA
 service.alarm.tersename=nasparkh
 service.alarm.label=SPARKTHRIFTDOWNALARM