Merge branch 'master' into SPARK-16861-refactor-pyspark-accumulator-api

Author: holdenk

core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala

@@ -26,11 +26,15 @@ import org.apache.spark.ui.{UIUtils, WebUIPage}
 private[ui] class EnvironmentPage(parent: EnvironmentTab) extends WebUIPage("") {
   private val listener = parent.listener
 
+  private def removePass(kv: (String, String)): (String, String) = {
+    if (kv._1.toLowerCase.contains("password")) (kv._1, "******") else kv
+  }
+
   def render(request: HttpServletRequest): Seq[Node] = {
     val runtimeInformationTable = UIUtils.listingTable(
       propertyHeader, jvmRow, listener.jvmInformation, fixedWidth = true)
     val sparkPropertiesTable = UIUtils.listingTable(
-      propertyHeader, propertyRow, listener.sparkProperties, fixedWidth = true)
+      propertyHeader, propertyRow, listener.sparkProperties.map(removePass), fixedWidth = true)
     val systemPropertiesTable = UIUtils.listingTable(
       propertyHeader, propertyRow, listener.systemProperties, fixedWidth = true)
     val classpathEntriesTable = UIUtils.listingTable(

core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala

@@ -611,7 +611,9 @@ private[spark] class ExternalSorter[K, V, C](
       val ds = deserializeStream
       deserializeStream = null
       fileStream = null
-      ds.close()
+      if (ds != null) {
+        ds.close()
+      }
       // NOTE: We don't do file.delete() here because that is done in ExternalSorter.stop().
       // This should also be fixed in ExternalAppendOnlyMap.
     }

dev/deps/spark-deps-hadoop-2.2

@@ -98,7 +98,7 @@ jersey-media-jaxb-2.22.2.jar
 jersey-server-2.22.2.jar
 jets3t-0.7.1.jar
 jetty-util-6.1.26.jar
-jline-2.12.jar
+jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar

dev/deps/spark-deps-hadoop-2.3

@@ -103,7 +103,7 @@ jersey-server-2.22.2.jar
 jets3t-0.9.3.jar
 jetty-6.1.26.jar
 jetty-util-6.1.26.jar
-jline-2.12.jar
+jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar

dev/deps/spark-deps-hadoop-2.4

@@ -103,7 +103,7 @@ jersey-server-2.22.2.jar
 jets3t-0.9.3.jar
 jetty-6.1.26.jar
 jetty-util-6.1.26.jar
-jline-2.12.jar
+jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar

dev/deps/spark-deps-hadoop-2.6

@@ -111,7 +111,7 @@ jersey-server-2.22.2.jar
 jets3t-0.9.3.jar
 jetty-6.1.26.jar
 jetty-util-6.1.26.jar
-jline-2.12.jar
+jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar

dev/deps/spark-deps-hadoop-2.7

@@ -111,7 +111,7 @@ jersey-server-2.22.2.jar
 jets3t-0.9.3.jar
 jetty-6.1.26.jar
 jetty-util-6.1.26.jar
-jline-2.12.jar
+jline-2.12.1.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar

pom.xml

@@ -159,8 +159,6 @@
     <commons.collections.version>3.2.2</commons.collections.version>
     <scala.version>2.11.8</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
-    <jline.version>${scala.version}</jline.version>
-    <jline.groupid>org.scala-lang</jline.groupid>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
     <fasterxml.jackson.version>2.6.5</fasterxml.jackson.version>
     <snappy.version>1.1.2.4</snappy.version>
@@ -1428,6 +1426,10 @@
             <groupId>org.codehaus.groovy</groupId>
             <artifactId>groovy-all</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>jline</groupId>
+            <artifactId>jline</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1832,6 +1834,11 @@
         <artifactId>antlr4-runtime</artifactId>
         <version>${antlr4.version}</version>
       </dependency>
+      <dependency>
+        <groupId>${jline.groupid}</groupId>
+        <artifactId>jline</artifactId>
+        <version>${jline.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -2538,15 +2545,6 @@
         <jline.version>${scala.version}</jline.version>
         <jline.groupid>org.scala-lang</jline.groupid>
       </properties>
-      <dependencyManagement>
-        <dependencies>
-          <dependency>
-            <groupId>${jline.groupid}</groupId>
-            <artifactId>jline</artifactId>
-            <version>${jline.version}</version>
-          </dependency>
-        </dependencies>
-      </dependencyManagement>
       <build>
         <plugins>
           <plugin>
@@ -2645,6 +2643,8 @@
       <properties>
         <scala.version>2.11.8</scala.version>
         <scala.binary.version>2.11</scala.binary.version>
+        <jline.version>2.12.1</jline.version>
+        <jline.groupid>jline</jline.groupid>
       </properties>
       <build>
         <plugins>

python/pyspark/ml/tuning.py

@@ -166,6 +166,8 @@ class CrossValidator(Estimator, ValidatorParams):
     >>> evaluator = BinaryClassificationEvaluator()
     >>> cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
     >>> cvModel = cv.fit(dataset)
+    >>> cvModel.avgMetrics[0]
+    0.5
     >>> evaluator.evaluate(cvModel.transform(dataset))
     0.8333...
 
@@ -234,7 +236,7 @@ def _fit(self, dataset):
                 model = est.fit(train, epm[j])
                 # TODO: duplicate evaluator to take extra params from input
                 metric = eva.evaluate(model.transform(validation, epm[j]))
-                metrics[j] += metric
+                metrics[j] += metric/nFolds
 
         if eva.isLargerBetter():
             bestIndex = np.argmax(metrics)

repl/pom.xml

@@ -71,6 +71,10 @@
       <version>${scala.version}</version>
     </dependency>
     <dependency>
+      <groupId>${jline.groupid}</groupId>
+      <artifactId>jline</artifactId>
+    </dependency>
+     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>jul-to-slf4j</artifactId>
     </dependency>
@@ -160,13 +164,6 @@
       <activation>
         <property><name>scala-2.10</name></property>
       </activation>
-      <dependencies>
-        <dependency>
-          <groupId>${jline.groupid}</groupId>
-          <artifactId>jline</artifactId>
-          <version>${jline.version}</version>
-        </dependency>
-      </dependencies>
     </profile>
 
     <profile>