Merge branch 'master' of https://github.com/apache/spark into SPARK-31167-missing-test-deps

Author: nchammas

R/pkg/DESCRIPTION

@@ -23,7 +23,7 @@ Suggests:
     testthat,
     e1071,
     survival,
-    arrow
+    arrow (>= 0.15.1)
 Collate:
     'schema.R'
     'generics.R'

appveyor.yml

@@ -48,7 +48,8 @@ install:
 build_script:
   # '-Djna.nosys=true' is required to avoid kernel32.dll load failure.
   # See SPARK-28759.
-  - cmd: mvn -DskipTests -Psparkr -Phive -Djna.nosys=true package
+  # Ideally we should check the tests related to Hive in SparkR as well (SPARK-31745).
+  - cmd: mvn -DskipTests -Psparkr -Djna.nosys=true package
 
 environment:
   NOT_CRAN: true

common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java

@@ -229,8 +229,6 @@ public class ShuffleMetrics implements MetricSet {
     private final Meter blockTransferRateBytes = new Meter();
     // Number of active connections to the shuffle service
     private Counter activeConnections = new Counter();
-    // Number of registered connections to the shuffle service
-    private Counter registeredConnections = new Counter();
     // Number of exceptions caught in connections to the shuffle service
     private Counter caughtExceptions = new Counter();
 
@@ -242,7 +240,6 @@ public ShuffleMetrics() {
       allMetrics.put("registeredExecutorsSize",
                      (Gauge<Integer>) () -> blockManager.getRegisteredExecutorsSize());
       allMetrics.put("numActiveConnections", activeConnections);
-      allMetrics.put("numRegisteredConnections", registeredConnections);
       allMetrics.put("numCaughtExceptions", caughtExceptions);
     }
 

common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java

@@ -198,6 +198,7 @@ protected void serviceInit(Configuration conf) throws Exception {
       // register metrics on the block handler into the Node Manager's metrics system.
       blockHandler.getAllMetrics().getMetrics().put("numRegisteredConnections",
           shuffleServer.getRegisteredConnections());
+      blockHandler.getAllMetrics().getMetrics().putAll(shuffleServer.getAllMetrics().getMetrics());
       YarnShuffleServiceMetrics serviceMetrics =
           new YarnShuffleServiceMetrics(blockHandler.getAllMetrics());
 

sql/core/src/main/scala/org/apache/spark/sql/execution/joins/package.scala renamed to common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java

@@ -15,17 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.execution
+package org.apache.spark.tags;
 
-/**
- * Physical execution operators for join operations.
- */
-package object joins {
-
-  sealed abstract class BuildSide
-
-  case object BuildRight extends BuildSide
+import java.lang.annotation.*;
 
-  case object BuildLeft extends BuildSide
+import org.scalatest.TagAnnotation;
 
-}
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface ChromeUITest { }

conf/spark-env.sh.template

@@ -62,6 +62,7 @@
 # Generic options for the daemons used in the standalone deploy mode
 # - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)
 # - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)
+# - SPARK_LOG_MAX_FILES Max log files of Spark daemons can rotate to. Default is 5.
 # - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)
 # - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)
 # - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)

core/src/main/java/org/apache/spark/SparkFirehoseListener.java

@@ -162,6 +162,11 @@ public void onSpeculativeTaskSubmitted(SparkListenerSpeculativeTaskSubmitted spe
     onEvent(speculativeTask);
   }
 
+  @Override
+  public void onResourceProfileAdded(SparkListenerResourceProfileAdded event) {
+    onEvent(event);
+  }
+
   @Override
   public void onOtherEvent(SparkListenerEvent event) {
     onEvent(event);