diff --git a/core/src/main/scala/org/apache/spark/Accumulable.scala b/core/src/main/scala/org/apache/spark/Accumulable.scala index 5532931e2a794..41c233e131df0 100644 --- a/core/src/main/scala/org/apache/spark/Accumulable.scala +++ b/core/src/main/scala/org/apache/spark/Accumulable.scala @@ -68,7 +68,9 @@ class Accumulable[R, T] private ( } private[spark] def this(initialValue: R, param: AccumulableParam[R, T], name: Option[String]) = { + // scalastyle:off this(initialValue, param, name, false /* countFailedValues */) + // scalastyle:on } def this(initialValue: R, param: AccumulableParam[R, T]) = this(initialValue, param, None) diff --git a/core/src/main/scala/org/apache/spark/FutureAction.scala b/core/src/main/scala/org/apache/spark/FutureAction.scala index 0899693988016..0185056350e50 100644 --- a/core/src/main/scala/org/apache/spark/FutureAction.scala +++ b/core/src/main/scala/org/apache/spark/FutureAction.scala @@ -174,7 +174,7 @@ trait JobSubmitter { * jobs. */ @DeveloperApi -class ComplexFutureAction[T](run : JobSubmitter => Future[T]) +class ComplexFutureAction[T](run: JobSubmitter => Future[T]) extends FutureAction[T] { self => @volatile private var _cancelled = false diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala index f83f5278e8b8f..37254696b23a1 100644 --- a/core/src/main/scala/org/apache/spark/Partitioner.scala +++ b/core/src/main/scala/org/apache/spark/Partitioner.scala @@ -105,7 +105,7 @@ class HashPartitioner(partitions: Int) extends Partitioner { * as the `partitions` parameter, in the case where the number of sampled records is less than * the value of `partitions`. */ -class RangePartitioner[K : Ordering : ClassTag, V]( +class RangePartitioner[K: Ordering: ClassTag, V]( partitions: Int, rdd: RDD[_ <: Product2[K, V]], private var ascending: Boolean = true) @@ -253,7 +253,7 @@ private[spark] object RangePartitioner { * @param sampleSizePerPartition max sample size per partition * @return (total number of items, an array of (partitionId, number of items, sample)) */ - def sketch[K : ClassTag]( + def sketch[K: ClassTag]( rdd: RDD[K], sampleSizePerPartition: Int): (Long, Array[(Int, Long, Array[K])]) = { val shift = rdd.id @@ -276,7 +276,7 @@ private[spark] object RangePartitioner { * @param partitions number of partitions * @return selected bounds */ - def determineBounds[K : Ordering : ClassTag]( + def determineBounds[K: Ordering: ClassTag]( candidates: ArrayBuffer[(K, Float)], partitions: Int): Array[K] = { val ordering = implicitly[Ordering[K]] diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala index 2480e56b72ccf..eabbaba81a425 100644 --- a/core/src/main/scala/org/apache/spark/SecurityManager.scala +++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala @@ -202,7 +202,7 @@ private[spark] class SecurityManager( stringToSet(sparkConf.get("spark.admin.acls", "")) // admin group acls should be set before view or modify group acls - private var adminAclsGroups : Set[String] = + private var adminAclsGroups: Set[String] = stringToSet(sparkConf.get("spark.admin.acls.groups", "")) private var viewAcls: Set[String] = _ diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala index de2f475c6895f..fa25994cf512b 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala @@ -595,7 +595,7 @@ private[spark] object SparkConf extends Logging { DeprecatedConfig("spark.executor.port", "2.0.0", "Not used any more") ) - Map(configs.map { cfg => (cfg.key -> cfg) } : _*) + Map(configs.map { cfg => (cfg.key -> cfg) }: _*) } /** diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index b2a26c51d4de1..1fefc3dfb30b0 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2869,7 +2869,7 @@ private[spark] class WritableFactory[T]( object WritableFactory { - private[spark] def simpleWritableFactory[T: ClassTag, W <: Writable : ClassTag](convert: T => W) + private[spark] def simpleWritableFactory[T: ClassTag, W <: Writable: ClassTag](convert: T => W) : WritableFactory[T] = { val writableClass = implicitly[ClassTag[W]].runtimeClass.asInstanceOf[Class[W]] new WritableFactory[T](_ => writableClass, convert) diff --git a/core/src/main/scala/org/apache/spark/TaskEndReason.scala b/core/src/main/scala/org/apache/spark/TaskEndReason.scala index a76283e33fa65..7f34fca4cfa97 100644 --- a/core/src/main/scala/org/apache/spark/TaskEndReason.scala +++ b/core/src/main/scala/org/apache/spark/TaskEndReason.scala @@ -192,7 +192,7 @@ private[spark] class ThrowableSerializationWrapper(var exception: Throwable) ext try { exception = in.readObject().asInstanceOf[Throwable] } catch { - case e : Exception => log.warn("Task exception could not be deserialized", e) + case e: Exception => log.warn("Task exception could not be deserialized", e) } } } diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala index f820401da2fc3..271261b1baf56 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala @@ -54,7 +54,7 @@ private[spark] object JavaUtils { override def iterator: ju.Iterator[ju.Map.Entry[A, B]] = new ju.Iterator[ju.Map.Entry[A, B]] { val ui = underlying.iterator - var prev : Option[A] = None + var prev: Option[A] = None def hasNext: Boolean = ui.hasNext @@ -65,7 +65,7 @@ private[spark] object JavaUtils { import scala.util.hashing.byteswap32 override def getKey: A = k override def getValue: B = v - override def setValue(v1 : B): B = self.put(k, v1) + override def setValue(v1: B): B = self.put(k, v1) override def hashCode: Int = byteswap32(k.hashCode) + (byteswap32(v.hashCode) << 16) override def equals(other: Any): Boolean = other match { case e: ju.Map.Entry[_, _] => k == e.getKey && v == e.getValue diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala index fb0405b1a69c6..7dae65c9f6066 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala @@ -396,7 +396,7 @@ private class PairwiseRDD(prev: RDD[Array[Byte]]) extends RDD[(Long, Array[Byte] case Seq(a, b) => (Utils.deserializeLongValue(a), b) case x => throw new SparkException("PairwiseRDD: unexpected value: " + x) } - val asJavaPairRDD : JavaPairRDD[Long, Array[Byte]] = JavaPairRDD.fromRDD(this) + val asJavaPairRDD: JavaPairRDD[Long, Array[Byte]] = JavaPairRDD.fromRDD(this) } private object SpecialLengths { @@ -867,7 +867,7 @@ private[spark] object PythonRDD extends Logging { private class BytesToString extends org.apache.spark.api.java.function.Function[Array[Byte], String] { - override def call(arr: Array[Byte]) : String = new String(arr, StandardCharsets.UTF_8) + override def call(arr: Array[Byte]): String = new String(arr, StandardCharsets.UTF_8) } /** diff --git a/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala b/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala index 86965dbc2e778..5204d7318d44b 100644 --- a/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala +++ b/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala @@ -95,7 +95,7 @@ private[python] class DoubleArrayToWritableConverter extends Converter[Any, Writ private[python] class WritableToDoubleArrayConverter extends Converter[Any, Array[Double]] { override def convert(obj: Any): Array[Double] = obj match { - case daw : DoubleArrayWritable => daw.get().map(_.asInstanceOf[DoubleWritable].get()) + case daw: DoubleArrayWritable => daw.get().map(_.asInstanceOf[DoubleWritable].get()) case other => throw new SparkException(s"Data of type $other is not supported") } } diff --git a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala index cfd37ac54ba23..818d2e60e03ad 100644 --- a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala +++ b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala @@ -164,7 +164,7 @@ private[r] class RBackendHandler(server: RBackend) throw new Exception(s"No matched method found for $cls.$methodName") } - val ret = selectedMethods(index.get).invoke(obj, args : _*) + val ret = selectedMethods(index.get).invoke(obj, args: _*) // Write status bit writeInt(dos, 0) @@ -185,7 +185,7 @@ private[r] class RBackendHandler(server: RBackend) throw new Exception(s"No matched constructor found for $cls") } - val obj = ctors(index.get).newInstance(args : _*) + val obj = ctors(index.get).newInstance(args: _*) writeInt(dos, 0) writeObject(dos, obj.asInstanceOf[AnyRef], server.jvmObjectTracker) diff --git a/core/src/main/scala/org/apache/spark/api/r/RRDD.scala b/core/src/main/scala/org/apache/spark/api/r/RRDD.scala index 295355c7bf018..adf96d769f298 100644 --- a/core/src/main/scala/org/apache/spark/api/r/RRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/r/RRDD.scala @@ -67,7 +67,7 @@ private class PairwiseRRDD[T: ClassTag]( parent, numPartitions, hashFunc, deserializer, SerializationFormats.BYTE, packageNames, broadcastVars.map(x => x.asInstanceOf[Broadcast[Object]])) { - lazy val asJavaPairRDD : JavaPairRDD[Int, Array[Byte]] = JavaPairRDD.fromRDD(this) + lazy val asJavaPairRDD: JavaPairRDD[Int, Array[Byte]] = JavaPairRDD.fromRDD(this) } /** @@ -83,7 +83,7 @@ private class RRDD[T: ClassTag]( extends BaseRRDD[T, Array[Byte]]( parent, -1, func, deserializer, serializer, packageNames, broadcastVars.map(x => x.asInstanceOf[Broadcast[Object]])) { - lazy val asJavaRDD : JavaRDD[Array[Byte]] = JavaRDD.fromRDD(this) + lazy val asJavaRDD: JavaRDD[Array[Byte]] = JavaRDD.fromRDD(this) } /** @@ -98,7 +98,7 @@ private class StringRRDD[T: ClassTag]( extends BaseRRDD[T, String]( parent, -1, func, deserializer, SerializationFormats.STRING, packageNames, broadcastVars.map(x => x.asInstanceOf[Broadcast[Object]])) { - lazy val asJavaRDD : JavaRDD[String] = JavaRDD.fromRDD(this) + lazy val asJavaRDD: JavaRDD[String] = JavaRDD.fromRDD(this) } private[r] object RRDD { diff --git a/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala b/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala index e88988fe03b2e..d82fadcd1715e 100644 --- a/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala +++ b/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala @@ -52,8 +52,8 @@ private[spark] class BroadcastManager( private val nextBroadcastId = new AtomicLong(0) - def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean): Broadcast[T] = { - broadcastFactory.newBroadcast[T](value_, isLocal, nextBroadcastId.getAndIncrement()) + def newBroadcast[T: ClassTag](_value: T, isLocal: Boolean): Broadcast[T] = { + broadcastFactory.newBroadcast[T](_value, isLocal, nextBroadcastId.getAndIncrement()) } def unbroadcast(id: Long, removeFromDriver: Boolean, blocking: Boolean) { diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala index b11f9ba171b84..52ca4493f1c45 100644 --- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala +++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala @@ -30,8 +30,8 @@ private[spark] class TorrentBroadcastFactory extends BroadcastFactory { override def initialize(isDriver: Boolean, conf: SparkConf, securityMgr: SecurityManager) { } - override def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean, id: Long): Broadcast[T] = { - new TorrentBroadcast[T](value_, id) + override def newBroadcast[T: ClassTag](_value: T, isLocal: Boolean, id: Long): Broadcast[T] = { + new TorrentBroadcast[T](_value, id) } override def stop() { } diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala index 8d491ddf6e092..cc1e4fe89ce8f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala +++ b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala @@ -89,7 +89,9 @@ class ExternalShuffleService(sparkConf: SparkConf, securityManager: SecurityMana /** Clean up all shuffle files associated with an application that has exited. */ def applicationRemoved(appId: String): Unit = { + // scalastyle:off blockHandler.applicationRemoved(appId, true /* cleanupLocalDirs */) + // scalastyle:on } def stop() { diff --git a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala index c6307da61c7eb..9764c46d6a6a2 100644 --- a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala +++ b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala @@ -406,7 +406,7 @@ private object SparkDocker { new TestWorkerInfo(ip, id, outFile) } - private def startNode(dockerCmd: ProcessBuilder) : (String, DockerId, File) = { + private def startNode(dockerCmd: ProcessBuilder): (String, DockerId, File) = { val ipPromise = Promise[String]() val outFile = File.createTempFile("fault-tolerance-test", "", Utils.createTempDir()) val outStream: FileWriter = new FileWriter(outFile) @@ -440,7 +440,7 @@ private object Docker extends Logging { cmd } - def kill(dockerId: DockerId) : Unit = { + def kill(dockerId: DockerId): Unit = { "docker kill %s".format(dockerId.id).! } diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala index a370526c46f3d..db47649c70a21 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala @@ -602,7 +602,7 @@ private[history] object ApplicationCacheCheckFilterRelay extends Logging { * @param cache new cache */ def setApplicationCache(cache: ApplicationCache): Unit = { - applicationCache.foreach( c => logWarning(s"Overwriting application cache $c")) + applicationCache.foreach(c => logWarning(s"Overwriting application cache $c")) applicationCache = Some(cache) } @@ -650,13 +650,13 @@ private[history] object ApplicationCacheCheckFilterRelay extends Logging { def registerFilter( ui: SparkUI, appId: String, - attemptId: Option[String] ): Unit = { + attemptId: Option[String]): Unit = { require(ui != null) val enumDispatcher = java.util.EnumSet.of(DispatcherType.ASYNC, DispatcherType.REQUEST) val holder = new FilterHolder() holder.setClassName(FILTER_NAME) holder.setInitParameter(APP_ID, appId) - attemptId.foreach( id => holder.setInitParameter(ATTEMPT_ID, id)) + attemptId.foreach(id => holder.setInitParameter(ATTEMPT_ID, id)) require(ui.getHandlers != null, "null handlers") ui.getHandlers.foreach { handler => handler.addFilter(holder, "/*", enumDispatcher) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index bed47455680dd..d5ce01a2ec588 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -114,7 +114,7 @@ private[deploy] class Worker( */ private var masterAddressToConnect: Option[RpcAddress] = None private var activeMasterUrl: String = "" - private[worker] var activeMasterWebUiUrl : String = "" + private[worker] var activeMasterWebUiUrl: String = "" private var workerWebUiUrl: String = "" private val workerUri = RpcEndpointAddress(rpcEnv.address, endpointName).toString private var registered = false @@ -178,7 +178,7 @@ private[deploy] class Worker( // This sporadically fails - not sure why ... !workDir.exists() && !workDir.mkdirs() // So attempting to create and then check if directory was created or not. workDir.mkdirs() - if ( !workDir.exists() || !workDir.isDirectory) { + if (!workDir.exists() || !workDir.isDirectory) { logError("Failed to create work directory " + workDir) System.exit(1) } diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 19e7eb086f413..1d45c32058d8e 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -205,7 +205,7 @@ private[spark] class Executor( * tasks instead of taking the JVM down. * @param interruptThread whether to interrupt the task thread */ - def killAllTasks(interruptThread: Boolean, reason: String) : Unit = { + def killAllTasks(interruptThread: Boolean, reason: String): Unit = { runningTasks.keys().asScala.foreach(t => killTask(t, interruptThread = interruptThread, reason = reason)) } diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala index d16f4a1fc4e3b..e634566a067d4 100644 --- a/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala +++ b/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala @@ -29,7 +29,7 @@ import org.apache.spark.metrics.source.Source private[spark] class ExecutorSource(threadPool: ThreadPoolExecutor, executorId: String) extends Source { - private def fileStats(scheme: String) : Option[FileSystem.Statistics] = + private def fileStats(scheme: String): Option[FileSystem.Statistics] = FileSystem.getAllStatistics.asScala.find(s => s.getScheme.equals(scheme)) private def registerFileSystemStat[T]( diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala index c7f2847731fcb..c4a13287b4367 100644 --- a/core/src/main/scala/org/apache/spark/internal/Logging.scala +++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala @@ -32,7 +32,7 @@ trait Logging { // Make the log field transient so that objects with Logging can // be serialized and used on another machine - @transient private var log_ : Logger = null + @transient private var _log: Logger = null // Method to get the logger name for this object protected def logName = { @@ -42,11 +42,11 @@ trait Logging { // Method to get or create the logger for this object protected def log: Logger = { - if (log_ == null) { + if (_log == null) { initializeLogIfNecessary(false) - log_ = LoggerFactory.getLogger(logName) + _log = LoggerFactory.getLogger(logName) } - log_ + _log } // Log methods that take only a String diff --git a/core/src/main/scala/org/apache/spark/launcher/LauncherBackend.scala b/core/src/main/scala/org/apache/spark/launcher/LauncherBackend.scala index a5d41a1eeb479..2fe79713c1e26 100644 --- a/core/src/main/scala/org/apache/spark/launcher/LauncherBackend.scala +++ b/core/src/main/scala/org/apache/spark/launcher/LauncherBackend.scala @@ -86,7 +86,7 @@ private[spark] abstract class LauncherBackend { /** * Callback for when the launcher handle disconnects from this backend. */ - protected def onDisconnected() : Unit = { } + protected def onDisconnected(): Unit = { } private def fireStopRequest(): Unit = { val thread = LauncherBackend.threadFactory.newThread(new Runnable() { diff --git a/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala b/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala index d2b4187df5d50..ae169b76529a8 100644 --- a/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala +++ b/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala @@ -26,7 +26,7 @@ import org.apache.spark.util.collection.OpenHashMap /** * An ApproximateEvaluator for counts by key. Returns a map of key to confidence interval. */ -private[spark] class GroupedCountEvaluator[T : ClassTag](totalOutputs: Int, confidence: Double) +private[spark] class GroupedCountEvaluator[T: ClassTag](totalOutputs: Int, confidence: Double) extends ApproximateEvaluator[OpenHashMap[T, Long], Map[T, BoundedDouble]] { private var outputsMerged = 0 diff --git a/core/src/main/scala/org/apache/spark/partial/PartialResult.scala b/core/src/main/scala/org/apache/spark/partial/PartialResult.scala index 25cb7490aa9c9..44c7d26ef3e7e 100644 --- a/core/src/main/scala/org/apache/spark/partial/PartialResult.scala +++ b/core/src/main/scala/org/apache/spark/partial/PartialResult.scala @@ -77,9 +77,9 @@ class PartialResult[R](initialVal: R, isFinal: Boolean) { /** * Transform this PartialResult into a PartialResult of type T. */ - def map[T](f: R => T) : PartialResult[T] = { + def map[T](f: R => T): PartialResult[T] = { new PartialResult[T](f(initialVal), isFinal) { - override def getFinalValue() : T = synchronized { + override def getFinalValue(): T = synchronized { f(PartialResult.this.getFinalValue()) } override def onComplete(handler: T => Unit): PartialResult[T] = synchronized { @@ -90,7 +90,7 @@ class PartialResult[R](initialVal: R, isFinal: Boolean) { PartialResult.this.onFail(handler) } } - override def toString : String = synchronized { + override def toString: String = synchronized { PartialResult.this.getFinalValueInternal() match { case Some(value) => "(final: " + f(value) + ")" case None => "(partial: " + initialValue + ")" diff --git a/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala b/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala index 57108dcedcf0c..f6ffec4d19b22 100644 --- a/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala @@ -48,8 +48,8 @@ class CartesianPartition( private[spark] class CartesianRDD[T: ClassTag, U: ClassTag]( sc: SparkContext, - var rdd1 : RDD[T], - var rdd2 : RDD[U]) + var rdd1: RDD[T], + var rdd2: RDD[U]) extends RDD[(T, U)](sc, Nil) with Serializable { diff --git a/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala index 2cba1febe8759..6e848dbeebb11 100644 --- a/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala @@ -379,7 +379,7 @@ private class DefaultPartitionCoalescer(val balanceSlack: Double = 0.10) } } - def getPartitions: Array[PartitionGroup] = groupArr.filter( pg => pg.numPartitions > 0).toArray + def getPartitions: Array[PartitionGroup] = groupArr.filter(pg => pg.numPartitions > 0).toArray /** * Runs the packing algorithm and returns an array of PartitionGroups that if possible are diff --git a/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala index 14331dfd0c987..fc3a29b95c6dc 100644 --- a/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/DoubleRDDFunctions.scala @@ -134,7 +134,7 @@ class DoubleRDDFunctions(self: RDD[Double]) extends Logging with Serializable { }.reduce { (maxmin1, maxmin2) => (maxmin1._1.max(maxmin2._1), maxmin1._2.min(maxmin2._2)) } - if (min.isNaN || max.isNaN || max.isInfinity || min.isInfinity ) { + if (min.isNaN || max.isNaN || max.isInfinity || min.isInfinity) { throw new UnsupportedOperationException( "Histogram on either an empty RDD or RDD containing +/-infinity or NaN") } diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala index 482875e6c1ac5..dce16eb996221 100644 --- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala @@ -67,7 +67,7 @@ private[spark] class NewHadoopPartition( */ @DeveloperApi class NewHadoopRDD[K, V]( - sc : SparkContext, + sc: SparkContext, inputFormatClass: Class[_ <: InputFormat[K, V]], keyClass: Class[K], valueClass: Class[V], diff --git a/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala index a5992022d0832..f14e9cb9c011c 100644 --- a/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/OrderedRDDFunctions.scala @@ -42,9 +42,9 @@ import org.apache.spark.internal.Logging * rdd.sortByKey() * }}} */ -class OrderedRDDFunctions[K : Ordering : ClassTag, +class OrderedRDDFunctions[K: Ordering: ClassTag, V: ClassTag, - P <: Product2[K, V] : ClassTag] @DeveloperApi() ( + P <: Product2[K, V]: ClassTag] @DeveloperApi() ( self: RDD[P]) extends Logging with Serializable { private val ordering = implicitly[Ordering[K]] diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 58762cc0838cd..9b38899e65a21 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -348,7 +348,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)]) map.put(pair._1, if (old == null) pair._2 else cleanedF(old, pair._2)) } Iterator(map) - } : Iterator[JHashMap[K, V]] + }: Iterator[JHashMap[K, V]] val mergeMaps = (m1: JHashMap[K, V], m2: JHashMap[K, V]) => { m2.asScala.foreach { pair => @@ -356,7 +356,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)]) m1.put(pair._1, if (old == null) pair._2 else cleanedF(old, pair._2)) } m1 - } : JHashMap[K, V] + }: JHashMap[K, V] self.mapPartitions(reducePartition).reduce(mergeMaps).asScala } @@ -546,7 +546,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)]) * (k, v2) is in `other`. Uses the given Partitioner to partition the output RDD. */ def join[W](other: RDD[(K, W)], partitioner: Partitioner): RDD[(K, (V, W))] = self.withScope { - this.cogroup(other, partitioner).flatMapValues( pair => + this.cogroup(other, partitioner).flatMapValues(pair => for (v <- pair._1.iterator; w <- pair._2.iterator) yield (v, w) ) } @@ -944,7 +944,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)]) buf += pair._2 } buf - } : Seq[V] + }: Seq[V] val res = self.context.runJob(self, process, Array(index)) res(0) case None => diff --git a/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala index ce75a16031a3f..19cafdac64982 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala @@ -38,7 +38,7 @@ private[spark] class PruneDependency[T](rdd: RDD[T], partitionFilterFunc: Int => @transient val partitions: Array[Partition] = rdd.partitions .filter(s => partitionFilterFunc(s.index)).zipWithIndex - .map { case(split, idx) => new PartitionPruningRDDPartition(idx, split) : Partition } + .map { case(split, idx) => new PartitionPruningRDDPartition(idx, split): Partition } override def getParents(partitionId: Int): List[Int] = { List(partitions(partitionId).asInstanceOf[PartitionPruningRDDPartition].parentSplit.index) diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index 2985c90119468..0b8c60fc9e581 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -223,8 +223,8 @@ abstract class RDD[T: ClassTag]( // Our dependencies and partitions will be gotten by calling subclass's methods below, and will // be overwritten when we're checkpointed - private var dependencies_ : Seq[Dependency[_]] = null - @transient private var partitions_ : Array[Partition] = null + private var _dependencies: Seq[Dependency[_]] = null + @transient private var _partitions: Array[Partition] = null /** An Option holding our checkpoint RDD, if we are checkpointed */ private def checkpointRDD: Option[CheckpointRDD[T]] = checkpointData.flatMap(_.checkpointRDD) @@ -235,10 +235,10 @@ abstract class RDD[T: ClassTag]( */ final def dependencies: Seq[Dependency[_]] = { checkpointRDD.map(r => List(new OneToOneDependency(r))).getOrElse { - if (dependencies_ == null) { - dependencies_ = getDependencies + if (_dependencies == null) { + _dependencies = getDependencies } - dependencies_ + _dependencies } } @@ -248,14 +248,14 @@ abstract class RDD[T: ClassTag]( */ final def partitions: Array[Partition] = { checkpointRDD.map(_.partitions).getOrElse { - if (partitions_ == null) { - partitions_ = getPartitions - partitions_.zipWithIndex.foreach { case (partition, index) => + if (_partitions == null) { + _partitions = getPartitions + _partitions.zipWithIndex.foreach { case (partition, index) => require(partition.index == index, s"partitions($index).partition == ${partition.index}, but it should equal $index") } } - partitions_ + _partitions } } @@ -455,7 +455,7 @@ abstract class RDD[T: ClassTag]( position = position + 1 (position, t) } - } : Iterator[(Int, T)] + }: Iterator[(Int, T)] // include a shuffle step so that our upstream tasks are still distributed new CoalescedRDD( @@ -533,7 +533,7 @@ abstract class RDD[T: ClassTag]( * @return A random sub-sample of the RDD without replacement. */ private[spark] def randomSampleWithRange(lb: Double, ub: Double, seed: Long): RDD[T] = { - this.mapPartitionsWithIndex( { (index, partition) => + this.mapPartitionsWithIndex({ (index, partition) => val sampler = new BernoulliCellSampler[T](lb, ub) sampler.setSeed(seed + index) sampler.sample(partition) @@ -1743,7 +1743,7 @@ abstract class RDD[T: ClassTag]( */ private[spark] def markCheckpointed(): Unit = { clearDependencies() - partitions_ = null + _partitions = null deps = null // Forget the constructor argument for dependencies too } @@ -1754,7 +1754,7 @@ abstract class RDD[T: ClassTag]( * logic. See [[org.apache.spark.rdd.UnionRDD]] for an example. */ protected def clearDependencies() { - dependencies_ = null + _dependencies = null } /** A description of this RDD and its recursive dependencies for debugging. */ @@ -1834,7 +1834,7 @@ abstract class RDD[T: ClassTag]( override def toString: String = "%s%s[%d] at %s".format( Option(name).map(_ + " ").getOrElse(""), getClass.getSimpleName, id, getCreationSite) - def toJavaRDD() : JavaRDD[T] = { + def toJavaRDD(): JavaRDD[T] = { new JavaRDD(this)(elementClassTag) } } @@ -1876,7 +1876,7 @@ object RDD { keyWritableFactory.writableClass(kt), valueWritableFactory.writableClass(vt)) } - implicit def rddToOrderedRDDFunctions[K : Ordering : ClassTag, V: ClassTag](rdd: RDD[(K, V)]) + implicit def rddToOrderedRDDFunctions[K: Ordering: ClassTag, V: ClassTag](rdd: RDD[(K, V)]) : OrderedRDDFunctions[K, V, (K, V)] = { new OrderedRDDFunctions[K, V, (K, V)](rdd) } diff --git a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala index 86a332790fb00..24aa81d9c5bc1 100644 --- a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala @@ -32,7 +32,7 @@ import org.apache.spark.internal.Logging * @note This can't be part of PairRDDFunctions because we need more implicit parameters to * convert our keys and values to Writable. */ -class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag]( +class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable: ClassTag]( self: RDD[(K, V)], _keyWritableClass: Class[_ <: Writable], _valueWritableClass: Class[_ <: Writable]) diff --git a/core/src/main/scala/org/apache/spark/rdd/WholeTextFileRDD.scala b/core/src/main/scala/org/apache/spark/rdd/WholeTextFileRDD.scala index 8e1baae796fc5..c31d755f65c73 100644 --- a/core/src/main/scala/org/apache/spark/rdd/WholeTextFileRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/WholeTextFileRDD.scala @@ -29,7 +29,7 @@ import org.apache.spark.input.WholeTextFileInputFormat * An RDD that reads a bunch of text files in, and each text file becomes one record. */ private[spark] class WholeTextFileRDD( - sc : SparkContext, + sc: SparkContext, inputFormatClass: Class[_ <: WholeTextFileInputFormat], keyClass: Class[Text], valueClass: Class[Text], diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 3422a5f204b12..784d7444dd0f9 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1077,7 +1077,7 @@ class DAGScheduler( s"(available: ${stage.isAvailable}," + s"available outputs: ${stage.numAvailableOutputs}," + s"partitions: ${stage.numPartitions})" - case stage : ResultStage => + case stage: ResultStage => s"Stage ${stage} is actually done; (partitions: ${stage.numPartitions})" } logDebug(debugString) diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala index 35690b2783ad3..df523556f9f4f 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala @@ -49,7 +49,7 @@ import org.apache.spark.util.{JsonProtocol, Utils} */ private[spark] class EventLoggingListener( appId: String, - appAttemptId : Option[String], + appAttemptId: Option[String], logBaseDir: URI, sparkConf: SparkConf, hadoopConf: Configuration) @@ -57,7 +57,7 @@ private[spark] class EventLoggingListener( import EventLoggingListener._ - def this(appId: String, appAttemptId : Option[String], logBaseDir: URI, sparkConf: SparkConf) = + def this(appId: String, appAttemptId: Option[String], logBaseDir: URI, sparkConf: SparkConf) = this(appId, appAttemptId, logBaseDir, sparkConf, SparkHadoopUtil.get.newConfiguration(sparkConf)) diff --git a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala index 290fd073caf27..d5723fc634b42 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala @@ -89,7 +89,7 @@ private[scheduler] abstract class Stage( */ val fetchFailedAttemptIds = new HashSet[Int] - private[scheduler] def clearFailures() : Unit = { + private[scheduler] def clearFailures(): Unit = { fetchFailedAttemptIds.clear() } diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala index a284f7956cd31..f6f01f5770223 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala @@ -126,7 +126,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul def enqueueFailedTask(taskSetManager: TaskSetManager, tid: Long, taskState: TaskState, serializedData: ByteBuffer) { - var reason : TaskFailedReason = UnknownReason + var reason: TaskFailedReason = UnknownReason try { getTaskResultExecutor.execute(new Runnable { override def run(): Unit = Utils.logUncaughtExceptions { diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 737b383631148..0e57a85c78ea5 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -275,7 +275,7 @@ private[spark] class TaskSchedulerImpl( maxLocality: TaskLocality, shuffledOffers: Seq[WorkerOffer], availableCpus: Array[Int], - tasks: IndexedSeq[ArrayBuffer[TaskDescription]]) : Boolean = { + tasks: IndexedSeq[ArrayBuffer[TaskDescription]]): Boolean = { var launchedTask = false // nodes and executors that are blacklisted for the entire application have already been // filtered out by this point diff --git a/core/src/main/scala/org/apache/spark/security/GroupMappingServiceProvider.scala b/core/src/main/scala/org/apache/spark/security/GroupMappingServiceProvider.scala index ea047a4f75d55..989e30505a694 100644 --- a/core/src/main/scala/org/apache/spark/security/GroupMappingServiceProvider.scala +++ b/core/src/main/scala/org/apache/spark/security/GroupMappingServiceProvider.scala @@ -33,6 +33,6 @@ trait GroupMappingServiceProvider { * @param userName User's Name * @return set of groups that the user belongs to. Empty in case of an invalid user. */ - def getGroups(userName : String) : Set[String] + def getGroups(userName: String): Set[String] } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala index 1279b281ad8d8..cc2705c0663d4 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllRDDResource.scala @@ -77,7 +77,7 @@ private[spark] object AllRDDResource { if (rddInfo.storageLevel.useOffHeap) status.memUsedByRdd(rddId) else 0L), onHeapMemoryRemaining = status.onHeapMemRemaining, offHeapMemoryRemaining = status.offHeapMemRemaining - ) } ) + ) }) } else { None } @@ -90,7 +90,7 @@ private[spark] object AllRDDResource { diskUsed = block.diskSize, executors = locations ) - } ) + }) } else { None } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala index 56028710ecc66..37c17842c3131 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala @@ -69,7 +69,7 @@ private[v1] object AllStagesResource { } val taskData = if (includeDetails) { - Some(stageUiData.taskData.map { case (k, v) => k -> convertTaskData(v) } ) + Some(stageUiData.taskData.map { case (k, v) => k -> convertTaskData(v) }) } else { None } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 05948f2661056..897784bf9ae1f 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -46,15 +46,15 @@ class ApplicationAttemptInfo private[spark]( } class ExecutorStageSummary private[spark]( - val taskTime : Long, - val failedTasks : Int, - val succeededTasks : Int, - val inputBytes : Long, - val outputBytes : Long, - val shuffleRead : Long, - val shuffleWrite : Long, - val memoryBytesSpilled : Long, - val diskBytesSpilled : Long) + val taskTime: Long, + val failedTasks: Int, + val succeededTasks: Int, + val inputBytes: Long, + val outputBytes: Long, + val shuffleRead: Long, + val shuffleWrite: Long, + val memoryBytesSpilled: Long, + val diskBytesSpilled: Long) class ExecutorSummary private[spark]( val id: String, diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala index 2c3da0ee85e06..b64c346324bfc 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala @@ -35,19 +35,19 @@ import org.apache.spark.util.Utils */ @DeveloperApi class BlockManagerId private ( - private var executorId_ : String, - private var host_ : String, - private var port_ : Int, - private var topologyInfo_ : Option[String]) + private var _executorId: String, + private var _host: String, + private var _port: Int, + private var _topologyInfo: Option[String]) extends Externalizable { private def this() = this(null, null, 0, None) // For deserialization only - def executorId: String = executorId_ + def executorId: String = _executorId - if (null != host_) { - Utils.checkHost(host_) - assert (port_ > 0) + if (null != _host) { + Utils.checkHost(_host) + assert (_port > 0) } def hostPort: String = { @@ -57,11 +57,11 @@ class BlockManagerId private ( host + ":" + port } - def host: String = host_ + def host: String = _host - def port: Int = port_ + def port: Int = _port - def topologyInfo: Option[String] = topologyInfo_ + def topologyInfo: Option[String] = _topologyInfo def isDriver: Boolean = { executorId == SparkContext.DRIVER_IDENTIFIER || @@ -69,20 +69,20 @@ class BlockManagerId private ( } override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException { - out.writeUTF(executorId_) - out.writeUTF(host_) - out.writeInt(port_) - out.writeBoolean(topologyInfo_.isDefined) + out.writeUTF(_executorId) + out.writeUTF(_host) + out.writeInt(_port) + out.writeBoolean(_topologyInfo.isDefined) // we only write topologyInfo if we have it topologyInfo.foreach(out.writeUTF(_)) } override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException { - executorId_ = in.readUTF() - host_ = in.readUTF() - port_ = in.readInt() + _executorId = in.readUTF() + _host = in.readUTF() + _port = in.readInt() val isTopologyInfoAvailable = in.readBoolean() - topologyInfo_ = if (isTopologyInfoAvailable) Option(in.readUTF()) else None + _topologyInfo = if (isTopologyInfoAvailable) Option(in.readUTF()) else None } @throws(classOf[IOException]) diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index edf328b5ae538..3624ea42d59ab 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -236,10 +236,10 @@ private[spark] object JettyUtils extends Logging { def addFilters(handlers: Seq[ServletContextHandler], conf: SparkConf) { val filters: Array[String] = conf.get("spark.ui.filters", "").split(',').map(_.trim()) filters.foreach { - case filter : String => + case filter: String => if (!filter.isEmpty) { logInfo("Adding filter: " + filter) - val holder : FilterHolder = new FilterHolder() + val holder: FilterHolder = new FilterHolder() holder.setClassName(filter) // Get any parameters for each filter conf.get("spark." + filter + ".params", "").split(',').map(_.trim()).toSet.foreach { diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index ba798df13c95d..7e0143887cd2e 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -518,7 +518,7 @@ private[spark] object UIUtils extends Logging { param } - def getTimeZoneOffset() : Int = + def getTimeZoneOffset(): Int = TimeZone.getDefault().getOffset(System.currentTimeMillis()) / 1000 / 60 /** diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index cce7a7611b420..6b855199c1e55 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -275,7 +275,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { desc = jobSortDesc ).table(page) } catch { - case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) => + case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
Error while rendering job table:
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
index 382a6f979f2e6..7513254caa74b 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
@@ -106,7 +106,7 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage
}
- private def createExecutorTable() : Seq[Node] = {
+ private def createExecutorTable(): Seq[Node] = {
// Make an executor-id -> address map
val executorIdToAddress = mutable.HashMap[String, String]()
listener.blockManagerIds.foreach { blockManagerId =>
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 6b3dadc333316..385bde1fdddb4 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -307,7 +307,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
)
(_taskTable, _taskTable.table(page))
} catch {
- case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
+ case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
val errorMessage =
Error while rendering stage table:
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index a28daf7f90451..b9ca28fc81155 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -95,7 +95,7 @@ private[ui] class StageTableBase(
parameterOtherTable
).table(page)
} catch {
- case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
+ case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
Error while rendering stage table:
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
index 9448baac096dc..54b60433a7d17 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
@@ -31,21 +31,21 @@ import org.apache.spark.util.collection.OpenHashSet
private[spark] object UIData {
class ExecutorSummary {
- var taskTime : Long = 0
- var failedTasks : Int = 0
- var succeededTasks : Int = 0
- var reasonToNumKilled : Map[String, Int] = Map.empty
- var inputBytes : Long = 0
- var inputRecords : Long = 0
- var outputBytes : Long = 0
- var outputRecords : Long = 0
- var shuffleRead : Long = 0
- var shuffleReadRecords : Long = 0
- var shuffleWrite : Long = 0
- var shuffleWriteRecords : Long = 0
- var memoryBytesSpilled : Long = 0
- var diskBytesSpilled : Long = 0
- var isBlacklisted : Int = 0
+ var taskTime: Long = 0
+ var failedTasks: Int = 0
+ var succeededTasks: Int = 0
+ var reasonToNumKilled: Map[String, Int] = Map.empty
+ var inputBytes: Long = 0
+ var inputRecords: Long = 0
+ var outputBytes: Long = 0
+ var outputRecords: Long = 0
+ var shuffleRead: Long = 0
+ var shuffleReadRecords: Long = 0
+ var shuffleWrite: Long = 0
+ var shuffleWriteRecords: Long = 0
+ var memoryBytesSpilled: Long = 0
+ var diskBytesSpilled: Long = 0
+ var isBlacklisted: Int = 0
}
class JobUIData(
@@ -91,7 +91,7 @@ private[spark] object UIData {
var outputBytes: Long = _
var outputRecords: Long = _
var shuffleReadTotalBytes: Long = _
- var shuffleReadRecords : Long = _
+ var shuffleReadRecords: Long = _
var shuffleWriteBytes: Long = _
var shuffleWriteRecords: Long = _
var memoryBytesSpilled: Long = _
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
index 317e0aa5ea25c..7a3556caee46e 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
@@ -77,7 +77,7 @@ private[ui] class RDDPage(parent: StorageTab) extends WebUIPage("rdd") {
blockSortDesc)
_blockTable.table(page)
} catch {
- case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
+ case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
{e.getMessage}
}
diff --git a/core/src/main/scala/org/apache/spark/util/CollectionsUtils.scala b/core/src/main/scala/org/apache/spark/util/CollectionsUtils.scala
index 85da2842e8ddb..40a3e9caa13c0 100644
--- a/core/src/main/scala/org/apache/spark/util/CollectionsUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/CollectionsUtils.scala
@@ -22,7 +22,7 @@ import java.util
import scala.reflect.{classTag, ClassTag}
private[spark] object CollectionsUtils {
- def makeBinarySearch[K : Ordering : ClassTag] : (Array[K], K) => Int = {
+ def makeBinarySearch[K: Ordering: ClassTag]: (Array[K], K) => Int = {
// For primitive keys, we can use the natural ordering. Otherwise, use the Ordering comparator.
classTag[K] match {
case ClassTag.Float =>
diff --git a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala b/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
index 31d230d0fec8e..16dcc3ea19ac6 100644
--- a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
+++ b/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
@@ -41,7 +41,7 @@ abstract class CompletionIterator[ +A, +I <: Iterator[A]](sub: I) extends Iterat
}
private[spark] object CompletionIterator {
- def apply[A, I <: Iterator[A]](sub: I, completionFunction: => Unit) : CompletionIterator[A, I] = {
+ def apply[A, I <: Iterator[A]](sub: I, completionFunction: => Unit): CompletionIterator[A, I] = {
new CompletionIterator[A, I](sub) {
def completion(): Unit = completionFunction
}
diff --git a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
index 76a56298aaebc..66d3de8576dee 100644
--- a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
@@ -97,7 +97,7 @@ private[spark] trait ListenerBus[L <: AnyRef, E] extends Logging {
*/
protected def doPostEvent(listener: L, event: E): Unit
- private[spark] def findListenersByClass[T <: L : ClassTag](): Seq[T] = {
+ private[spark] def findListenersByClass[T <: L: ClassTag](): Seq[T] = {
val c = implicitly[ClassTag[T]].runtimeClass
listeners.asScala.filter(_.getClass == c).map(_.asInstanceOf[T]).toSeq
}
diff --git a/core/src/main/scala/org/apache/spark/util/MutablePair.scala b/core/src/main/scala/org/apache/spark/util/MutablePair.scala
index 3d95b7869f494..3c25b415f83b8 100644
--- a/core/src/main/scala/org/apache/spark/util/MutablePair.scala
+++ b/core/src/main/scala/org/apache/spark/util/MutablePair.scala
@@ -28,8 +28,8 @@ import org.apache.spark.annotation.DeveloperApi
* @param _2 Element 2 of this MutablePair
*/
@DeveloperApi
-case class MutablePair[@specialized(Int, Long, Double, Char, Boolean/* , AnyRef */) T1,
- @specialized(Int, Long, Double, Char, Boolean/* , AnyRef */) T2]
+case class MutablePair[@specialized(Int, Long, Double, Char, Boolean) T1,
+ @specialized(Int, Long, Double, Char, Boolean) T2]
(var _1: T1, var _2: T2)
extends Product2[T1, T2]
{
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index bbb7999e2a144..2b7558fdda303 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -162,7 +162,7 @@ private[spark] object Utils extends Logging {
}
/** Deserialize a Long value (used for [[org.apache.spark.api.python.PythonPartitioner]]) */
- def deserializeLongValue(bytes: Array[Byte]) : Long = {
+ def deserializeLongValue(bytes: Array[Byte]): Long = {
// Note: we assume that we are given a Long value encoded in network (big-endian) byte order
var result = bytes(7) & 0xFFL
result = result + ((bytes(6) & 0xFFL) << 8)
@@ -2068,7 +2068,7 @@ private[spark] object Utils extends Logging {
*/
def updateSparkConfigFromProperties(
conf: SparkConf,
- properties: Map[String, String]) : Unit = {
+ properties: Map[String, String]): Unit = {
properties.filter { case (k, v) =>
k.startsWith("spark.")
}.foreach { case (k, v) =>
diff --git a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
index e63e0e3e1f68f..6f9ab78f0809c 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
@@ -75,7 +75,7 @@ class BitSet(numBits: Int) extends Serializable {
assert(newBS.numWords >= numWords)
assert(newBS.numWords >= other.numWords)
var ind = 0
- while( ind < smaller ) {
+ while(ind < smaller) {
newBS.words(ind) = words(ind) & other.words(ind)
ind += 1
}
@@ -92,15 +92,15 @@ class BitSet(numBits: Int) extends Serializable {
assert(newBS.numWords >= other.numWords)
val smaller = math.min(numWords, other.numWords)
var ind = 0
- while( ind < smaller ) {
+ while(ind < smaller) {
newBS.words(ind) = words(ind) | other.words(ind)
ind += 1
}
- while( ind < numWords ) {
+ while(ind < numWords) {
newBS.words(ind) = words(ind)
ind += 1
}
- while( ind < other.numWords ) {
+ while(ind < other.numWords) {
newBS.words(ind) = other.words(ind)
ind += 1
}
@@ -120,10 +120,10 @@ class BitSet(numBits: Int) extends Serializable {
ind += 1
}
if (ind < numWords) {
- Array.copy( words, ind, newBS.words, ind, numWords - ind )
+ Array.copy(words, ind, newBS.words, ind, numWords - ind)
}
if (ind < other.numWords) {
- Array.copy( other.words, ind, newBS.words, ind, other.numWords - ind )
+ Array.copy(other.words, ind, newBS.words, ind, other.numWords - ind)
}
newBS
}
@@ -141,7 +141,7 @@ class BitSet(numBits: Int) extends Serializable {
ind += 1
}
if (ind < numWords) {
- Array.copy( words, ind, newBS.words, ind, numWords - ind )
+ Array.copy(words, ind, newBS.words, ind, numWords - ind)
}
newBS
}
diff --git a/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala b/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala
index 10ab0b3f89964..0d81f4f746bfd 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala
@@ -30,7 +30,7 @@ import scala.reflect.ClassTag
* distinguish between the 0/0.0/0L and non-exist value
*/
private[spark]
-class OpenHashMap[K : ClassTag, @specialized(Long, Int, Double) V: ClassTag](
+class OpenHashMap[K: ClassTag, @specialized(Long, Int, Double) V: ClassTag](
initialCapacity: Int)
extends Iterable[(K, V)]
with Serializable {
diff --git a/core/src/main/scala/org/apache/spark/util/collection/SortDataFormat.scala b/core/src/main/scala/org/apache/spark/util/collection/SortDataFormat.scala
index 9a7a5a4e74868..c6434e4580b17 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/SortDataFormat.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/SortDataFormat.scala
@@ -83,7 +83,7 @@ abstract class SortDataFormat[K, Buffer] {
* when the keys and values are not the same type.
*/
private[spark]
-class KVArraySortDataFormat[K, T <: AnyRef : ClassTag] extends SortDataFormat[K, Array[T]] {
+class KVArraySortDataFormat[K, T <: AnyRef: ClassTag] extends SortDataFormat[K, Array[T]] {
override def getKey(data: Array[T], pos: Int): K = data(2 * pos).asInstanceOf[K]
diff --git a/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala b/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala
index 3990ee1ec326d..78c0069c2de77 100644
--- a/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/AccumulatorSuite.scala
@@ -47,15 +47,15 @@ class AccumulatorSuite extends SparkFunSuite with Matchers with LocalSparkContex
implicit def setAccum[A]: AccumulableParam[mutable.Set[A], A] =
new AccumulableParam[mutable.Set[A], A] {
- def addInPlace(t1: mutable.Set[A], t2: mutable.Set[A]) : mutable.Set[A] = {
+ def addInPlace(t1: mutable.Set[A], t2: mutable.Set[A]): mutable.Set[A] = {
t1 ++= t2
t1
}
- def addAccumulator(t1: mutable.Set[A], t2: A) : mutable.Set[A] = {
+ def addAccumulator(t1: mutable.Set[A], t2: A): mutable.Set[A] = {
t1 += t2
t1
}
- def zero(t: mutable.Set[A]) : mutable.Set[A] = {
+ def zero(t: mutable.Set[A]): mutable.Set[A] = {
new mutable.HashSet[A]()
}
}
diff --git a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
index 6724af952505f..c1641bab7a074 100644
--- a/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ContextCleanerSuite.scala
@@ -523,14 +523,14 @@ class CleanerTester(
toBeCheckpointIds.synchronized { toBeCheckpointIds.isEmpty }
private def getRDDBlocks(rddId: Int): Seq[BlockId] = {
- blockManager.master.getMatchingBlockIds( _ match {
+ blockManager.master.getMatchingBlockIds(_ match {
case RDDBlockId(`rddId`, _) => true
case _ => false
}, askSlaves = true)
}
private def getShuffleBlocks(shuffleId: Int): Seq[BlockId] = {
- blockManager.master.getMatchingBlockIds( _ match {
+ blockManager.master.getMatchingBlockIds(_ match {
case ShuffleBlockId(`shuffleId`, _, _) => true
case ShuffleIndexBlockId(`shuffleId`, _, _) => true
case _ => false
@@ -538,7 +538,7 @@ class CleanerTester(
}
private def getBroadcastBlocks(broadcastId: Long): Seq[BlockId] = {
- blockManager.master.getMatchingBlockIds( _ match {
+ blockManager.master.getMatchingBlockIds(_ match {
case BroadcastBlockId(`broadcastId`, _) => true
case _ => false
}, askSlaves = true)
diff --git a/core/src/test/scala/org/apache/spark/Smuggle.scala b/core/src/test/scala/org/apache/spark/Smuggle.scala
index 9d9217ea1b485..24bd2242e73fb 100644
--- a/core/src/test/scala/org/apache/spark/Smuggle.scala
+++ b/core/src/test/scala/org/apache/spark/Smuggle.scala
@@ -62,7 +62,7 @@ object Smuggle {
private val lock = new ReentrantReadWriteLock
private val smuggledObjects = mutable.WeakHashMap.empty[Symbol, Any]
- private def get[T](key: Symbol) : T = {
+ private def get[T](key: Symbol): T = {
lock.readLock().lock()
try {
smuggledObjects(key).asInstanceOf[T]
@@ -78,6 +78,6 @@ object Smuggle {
* @tparam T
* @return the smuggled object represented by the wrapper.
*/
- implicit def unpackSmuggledObject[T](smuggle : Smuggle[T]): T = smuggle.smuggledObject
+ implicit def unpackSmuggledObject[T](smuggle: Smuggle[T]): T = smuggle.smuggledObject
}
diff --git a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
index 55a541d60ea3c..b21aea95b0ce7 100644
--- a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
@@ -30,7 +30,7 @@ private[deploy] object DeployTestUtils {
new ApplicationDescription("name", Some(4), 1234, cmd, "appUiUrl")
}
- def createAppInfo() : ApplicationInfo = {
+ def createAppInfo(): ApplicationInfo = {
val appDesc = createAppDesc()
val appInfo = new ApplicationInfo(JsonConstants.appInfoStartTime,
"id", appDesc, JsonConstants.submitDate, null, Int.MaxValue)
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index b089357e7b868..0c4b23aff1cbe 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -64,7 +64,7 @@ trait TestPrematureExit {
private[spark] def testPrematureExit(
input: Array[String],
searchString: String,
- mainObject: CommandLineUtils = SparkSubmit) : Unit = {
+ mainObject: CommandLineUtils = SparkSubmit): Unit = {
val printStream = new BufferPrintStream()
mainObject.printStream = printStream
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
index 871c87415d35d..f98ea0c996215 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
@@ -77,7 +77,7 @@ class ApplicationCacheSuite extends SparkFunSuite with Logging with MockitoSugar
override def getAppUI(appId: String, attemptId: Option[String]): Option[LoadedAppUI] = {
logDebug(s"getAppUI($appId, $attemptId)")
getAppUICount += 1
- instances.get(CacheKey(appId, attemptId)).map( e =>
+ instances.get(CacheKey(appId, attemptId)).map(e =>
LoadedAppUI(e.ui, updateProbe(appId, attemptId, e.probeTime)))
}
@@ -393,7 +393,7 @@ class ApplicationCacheSuite extends SparkFunSuite with Logging with MockitoSugar
val ids = new ListBuffer[String]()
// build a list of applications
val count = 100
- for (i <- 1 to count ) {
+ for (i <- 1 to count) {
val appId = f"app-$i%04d"
ids += appId
clock.advance(10)
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
index ce212a7513310..46dc3ab42f584 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
@@ -30,7 +30,7 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
import org.apache.spark.deploy.DeployTestUtils._
def cmd(javaOpts: String*): Command = {
- Command("", Seq.empty, Map.empty, Seq.empty, Seq.empty, Seq(javaOpts : _*))
+ Command("", Seq.empty, Map.empty, Seq.empty, Seq.empty, Seq(javaOpts: _*))
}
def conf(opts: (String, String)*): SparkConf = new SparkConf(loadDefaults = false).setAll(opts)
diff --git a/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
index 05013fbc49b8e..380074809de9b 100644
--- a/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
@@ -78,7 +78,7 @@ class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkCont
() => { DriverManager.getConnection("jdbc:derby:target/JdbcRDDSuiteDb") },
"SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?",
1, 100, 3,
- (r: ResultSet) => { r.getInt(1) } ).cache()
+ (r: ResultSet) => { r.getInt(1) }).cache()
assert(rdd.count === 100)
assert(rdd.reduce(_ + _) === 10100)
@@ -91,7 +91,7 @@ class JdbcRDDSuite extends SparkFunSuite with BeforeAndAfter with LocalSparkCont
() => { DriverManager.getConnection("jdbc:derby:target/JdbcRDDSuiteDb") },
"SELECT DATA FROM BIGINT_TEST WHERE ? <= ID AND ID <= ?",
1131544775L, 567279358897692673L, 20,
- (r: ResultSet) => { r.getInt(1) } ).cache()
+ (r: ResultSet) => { r.getInt(1) }).cache()
assert(rdd.count === 100)
assert(rdd.reduce(_ + _) === 5050)
}
diff --git a/core/src/test/scala/org/apache/spark/rdd/ZippedPartitionsSuite.scala b/core/src/test/scala/org/apache/spark/rdd/ZippedPartitionsSuite.scala
index 5d7b973fbd9ac..95755ceac124b 100644
--- a/core/src/test/scala/org/apache/spark/rdd/ZippedPartitionsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/ZippedPartitionsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.rdd
import org.apache.spark.{SharedSparkContext, SparkFunSuite}
object ZippedPartitionsSuite {
- def procZippedData(i: Iterator[Int], s: Iterator[String], d: Iterator[Double]) : Iterator[Int] = {
+ def procZippedData(i: Iterator[Int], s: Iterator[String], d: Iterator[Double]): Iterator[Int] = {
Iterator(i.toArray.size, s.toArray.size, d.toArray.size)
}
}
diff --git a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
index 59d8c14d74e30..c26b702047ee4 100644
--- a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
@@ -745,18 +745,18 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
// Construct RpcTimeout with a single property
val rt1 = RpcTimeout(conf, testProp)
- assert( testDurationSeconds === rt1.duration.toSeconds )
+ assert(testDurationSeconds === rt1.duration.toSeconds )
// Construct RpcTimeout with prioritized list of properties
val rt2 = RpcTimeout(conf, Seq("spark.ask.invalid.timeout", testProp, secondaryProp), "1s")
- assert( testDurationSeconds === rt2.duration.toSeconds )
+ assert(testDurationSeconds === rt2.duration.toSeconds )
// Construct RpcTimeout with default value,
val defaultProp = "spark.ask.default.timeout"
val defaultDurationSeconds = 1
val rt3 = RpcTimeout(conf, Seq(defaultProp), defaultDurationSeconds.toString + "s")
- assert( defaultDurationSeconds === rt3.duration.toSeconds )
- assert( rt3.timeoutProp.contains(defaultProp) )
+ assert(defaultDurationSeconds === rt3.duration.toSeconds )
+ assert(rt3.timeoutProp.contains(defaultProp) )
// Try to construct RpcTimeout with an unconfigured property
intercept[NoSuchElementException] {
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 453be26ed8d0c..40809771b1241 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -760,7 +760,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with Timeou
// when we abort the stage. This message will also be consumed by the EventLoggingListener
// so this will propagate up to the user.
var ended = false
- var jobResult : JobResult = null
+ var jobResult: JobResult = null
class EndListener extends SparkListener {
override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = {
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
index 80fb674725814..ef38963201db4 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
@@ -85,8 +85,10 @@ object FakeRackUtil {
* a list of "live" executors and their hostnames for isExecutorAlive and hasExecutorsAliveOnHost
* to work, and these are required for locality in TaskSetManager.
*/
+// scalastyle:off
class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* execId, host */)
extends TaskSchedulerImpl(sc)
+// scalastyle:on
{
val startedTasks = new ArrayBuffer[Long]
val endedTasks = new mutable.HashMap[Long, TaskEndReason]
diff --git a/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala
index 3734f1cb408fe..00d5c378119c4 100644
--- a/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/GenericAvroSerializerSuite.scala
@@ -29,7 +29,7 @@ import org.apache.spark.{SharedSparkContext, SparkFunSuite}
class GenericAvroSerializerSuite extends SparkFunSuite with SharedSparkContext {
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
- val schema : Schema = SchemaBuilder
+ val schema: Schema = SchemaBuilder
.record("testRecord").fields()
.requiredString("data")
.endRecord()
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 7c3922e47fbb9..56b5a89dc5a7f 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -268,19 +268,19 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
}
test("kryo with parallelize for specialized tuples") {
- assert (sc.parallelize( Array((1, 11), (2, 22), (3, 33)) ).count === 3)
+ assert (sc.parallelize(Array((1, 11), (2, 22), (3, 33))).count === 3)
}
test("kryo with parallelize for primitive arrays") {
- assert (sc.parallelize( Array(1, 2, 3) ).count === 3)
+ assert (sc.parallelize(Array(1, 2, 3)).count === 3)
}
test("kryo with collect for specialized tuples") {
- assert (sc.parallelize( Array((1, 11), (2, 22), (3, 33)) ).collect().head === (1, 11))
+ assert (sc.parallelize(Array((1, 11), (2, 22), (3, 33))).collect().head === (1, 11))
}
test("kryo with SerializableHyperLogLog") {
- assert(sc.parallelize( Array(1, 2, 3, 2, 3, 3, 2, 3, 1) ).countApproxDistinct(0.01) === 3)
+ assert(sc.parallelize(Array(1, 2, 3, 2, 3, 3, 2, 3, 1)).countApproxDistinct(0.01) === 3)
}
test("kryo with reduce") {
diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
index 85ccb33471048..ea57f9f03fd0c 100644
--- a/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriterSuite.scala
@@ -140,7 +140,9 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
conf
)
writer.write(Iterator.empty)
- writer.stop( /* success = */ true)
+ // scalastyle:off
+ writer.stop(/* success = */ true)
+ // scalastyle:on
assert(writer.getPartitionLengths.sum === 0)
assert(outputFile.exists())
assert(outputFile.length() === 0)
@@ -164,7 +166,9 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
conf
)
writer.write(records)
- writer.stop( /* success = */ true)
+ // scalastyle:off
+ writer.stop(/* success = */ true)
+ // scalastyle:on
assert(temporaryFilesCreated.nonEmpty)
assert(writer.getPartitionLengths.sum === outputFile.length())
assert(writer.getPartitionLengths.count(_ == 0L) === 4) // should be 4 zero length files
@@ -206,8 +210,9 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
assert(temporaryFilesCreated.nonEmpty)
// Only 3 temp shuffle files will be created
assert(temporaryFilesCreated.count(_.exists()) === 3)
-
- writer.stop( /* success = */ false)
+ // scalastyle:off
+ writer.stop(/* success = */ false)
+ // scalastyle:on
assert(temporaryFilesCreated.count(_.exists()) === 0) // check that temporary files were deleted
}
@@ -229,7 +234,9 @@ class BypassMergeSortShuffleWriterSuite extends SparkFunSuite with BeforeAndAfte
}))
}
assert(temporaryFilesCreated.nonEmpty)
- writer.stop( /* success = */ false)
+ // scalastyle:off
+ writer.stop(/* success = */ false)
+ // scalastyle:on
assert(temporaryFilesCreated.count(_.exists()) === 0)
}
diff --git a/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala
index b21c91f75d5c7..a8cd3ddec8229 100644
--- a/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala
@@ -36,7 +36,7 @@ class FlatmapIteratorSuite extends SparkFunSuite with LocalSparkContext {
sc = new SparkContext(sconf)
val expand_size = 100
val data = sc.parallelize((1 to 5).toSeq).
- flatMap( x => Stream.range(0, expand_size))
+ flatMap(x => Stream.range(0, expand_size))
var persisted = data.persist(StorageLevel.DISK_ONLY)
assert(persisted.count()===500)
assert(persisted.filter(_==1).count()===5)
diff --git a/core/src/test/scala/org/apache/spark/ui/jobs/JobProgressListenerSuite.scala b/core/src/test/scala/org/apache/spark/ui/jobs/JobProgressListenerSuite.scala
index 48be3be81755a..b54949706ef08 100644
--- a/core/src/test/scala/org/apache/spark/ui/jobs/JobProgressListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/jobs/JobProgressListenerSuite.scala
@@ -84,7 +84,7 @@ class JobProgressListenerSuite extends SparkFunSuite with LocalSparkContext with
}
test("test LRU eviction of stages") {
- def runWithListener(listener: JobProgressListener) : Unit = {
+ def runWithListener(listener: JobProgressListener): Unit = {
for (i <- 1 to 50) {
listener.onStageSubmitted(createStageStartEvent(i))
listener.onStageCompleted(createStageEndEvent(i))
diff --git a/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala b/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala
index 4920b7ee8bfb4..54738a41a4f0d 100644
--- a/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/ClosureCleanerSuite.scala
@@ -312,11 +312,11 @@ private object TestUserClosuresActuallyCleaned {
// Test SparkContext runJob
def testRunJob1(sc: SparkContext): Unit = {
val rdd = sc.parallelize(1 to 10, 10)
- sc.runJob(rdd, { (ctx: TaskContext, iter: Iterator[Int]) => return; 1 } )
+ sc.runJob(rdd, { (ctx: TaskContext, iter: Iterator[Int]) => return; 1 })
}
def testRunJob2(sc: SparkContext): Unit = {
val rdd = sc.parallelize(1 to 10, 10)
- sc.runJob(rdd, { iter: Iterator[Int] => return; 1 } )
+ sc.runJob(rdd, { iter: Iterator[Int] => return; 1 })
}
def testRunApproximateJob(sc: SparkContext): Unit = {
val rdd = sc.parallelize(1 to 10, 10)
diff --git a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
index 7e2da8e141532..40553314a2f90 100644
--- a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
@@ -65,7 +65,7 @@ class FileAppenderSuite extends SparkFunSuite with BeforeAndAfter with Logging {
val rolloverIntervalMillis = 100
val durationMillis = 1000
val numRollovers = durationMillis / rolloverIntervalMillis
- val textToAppend = (1 to numRollovers).map( _.toString * 10 )
+ val textToAppend = (1 to numRollovers).map(_.toString * 10 )
val appender = new RollingFileAppender(testInputStream, testFile,
new TimeBasedRollingPolicy(rolloverIntervalMillis, s"--HH-mm-ss-SSSS", false),
@@ -81,7 +81,7 @@ class FileAppenderSuite extends SparkFunSuite with BeforeAndAfter with Logging {
val rolloverIntervalMillis = 100
val durationMillis = 1000
val numRollovers = durationMillis / rolloverIntervalMillis
- val textToAppend = (1 to numRollovers).map( _.toString * 10 )
+ val textToAppend = (1 to numRollovers).map(_.toString * 10 )
val sparkConf = new SparkConf()
sparkConf.set("spark.executor.logs.rolling.enableCompression", "true")
@@ -98,7 +98,7 @@ class FileAppenderSuite extends SparkFunSuite with BeforeAndAfter with Logging {
val testOutputStream = new PipedOutputStream()
val testInputStream = new PipedInputStream(testOutputStream, 100 * 1000)
val rolloverSize = 1000
- val textToAppend = (1 to 3).map( _.toString * 1000 )
+ val textToAppend = (1 to 3).map(_.toString * 1000 )
val appender = new RollingFileAppender(testInputStream, testFile,
new SizeBasedRollingPolicy(rolloverSize, false), new SparkConf(), 99)
@@ -115,7 +115,7 @@ class FileAppenderSuite extends SparkFunSuite with BeforeAndAfter with Logging {
val testOutputStream = new PipedOutputStream()
val testInputStream = new PipedInputStream(testOutputStream, 100 * 1000)
val rolloverSize = 1000
- val textToAppend = (1 to 3).map( _.toString * 1000 )
+ val textToAppend = (1 to 3).map(_.toString * 1000 )
val sparkConf = new SparkConf()
sparkConf.set("spark.executor.logs.rolling.enableCompression", "true")
diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
index 2695295d451d5..7174899e08135 100644
--- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
@@ -52,7 +52,7 @@ class DummyClass7 {
}
object DummyString {
- def apply(str: String) : DummyString = new DummyString(str.toArray)
+ def apply(str: String): DummyString = new DummyString(str.toArray)
}
class DummyString(val arr: Array[Char]) {
override val hashCode: Int = 0
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index f7bc8f888b0d5..4311048793385 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -384,7 +384,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
}
test("deserialize long value") {
- val testval : Long = 9730889947L
+ val testval: Long = 9730889947L
val bbuf = ByteBuffer.allocate(8)
assert(bbuf.hasArray)
bbuf.order(ByteOrder.BIG_ENDIAN)
diff --git a/core/src/test/scala/org/apache/spark/util/collection/BitSetSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/BitSetSuite.scala
index 0169c9926e68f..d516f9cdf8906 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/BitSetSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/BitSetSuite.scala
@@ -70,13 +70,13 @@ class BitSetSuite extends SparkFunSuite {
assert(bitset.nextSetBit(97) === -1)
}
- test( "xor len(bitsetX) < len(bitsetY)" ) {
- val setBitsX = Seq( 0, 2, 3, 37, 41 )
- val setBitsY = Seq( 0, 1, 3, 37, 38, 41, 85)
+ test("xor len(bitsetX) < len(bitsetY)") {
+ val setBitsX = Seq(0, 2, 3, 37, 41 )
+ val setBitsY = Seq(0, 1, 3, 37, 38, 41, 85)
val bitsetX = new BitSet(60)
- setBitsX.foreach( i => bitsetX.set(i))
+ setBitsX.foreach(i => bitsetX.set(i))
val bitsetY = new BitSet(100)
- setBitsY.foreach( i => bitsetY.set(i))
+ setBitsY.foreach(i => bitsetY.set(i))
val bitsetXor = bitsetX ^ bitsetY
@@ -92,13 +92,13 @@ class BitSetSuite extends SparkFunSuite {
}
- test( "xor len(bitsetX) > len(bitsetY)" ) {
- val setBitsX = Seq( 0, 1, 3, 37, 38, 41, 85)
- val setBitsY = Seq( 0, 2, 3, 37, 41)
+ test("xor len(bitsetX) > len(bitsetY)") {
+ val setBitsX = Seq(0, 1, 3, 37, 38, 41, 85)
+ val setBitsY = Seq(0, 2, 3, 37, 41)
val bitsetX = new BitSet(100)
- setBitsX.foreach( i => bitsetX.set(i))
+ setBitsX.foreach(i => bitsetX.set(i))
val bitsetY = new BitSet(60)
- setBitsY.foreach( i => bitsetY.set(i))
+ setBitsY.foreach(i => bitsetY.set(i))
val bitsetXor = bitsetX ^ bitsetY
@@ -114,15 +114,15 @@ class BitSetSuite extends SparkFunSuite {
}
- test( "andNot len(bitsetX) < len(bitsetY)" ) {
- val setBitsX = Seq( 0, 2, 3, 37, 41, 48 )
- val setBitsY = Seq( 0, 1, 3, 37, 38, 41, 85)
+ test("andNot len(bitsetX) < len(bitsetY)") {
+ val setBitsX = Seq(0, 2, 3, 37, 41, 48 )
+ val setBitsY = Seq(0, 1, 3, 37, 38, 41, 85)
val bitsetX = new BitSet(60)
- setBitsX.foreach( i => bitsetX.set(i))
+ setBitsX.foreach(i => bitsetX.set(i))
val bitsetY = new BitSet(100)
- setBitsY.foreach( i => bitsetY.set(i))
+ setBitsY.foreach(i => bitsetY.set(i))
- val bitsetDiff = bitsetX.andNot( bitsetY )
+ val bitsetDiff = bitsetX.andNot(bitsetY )
assert(bitsetDiff.nextSetBit(0) === 2)
assert(bitsetDiff.nextSetBit(1) === 2)
@@ -133,15 +133,15 @@ class BitSetSuite extends SparkFunSuite {
assert(bitsetDiff.nextSetBit(65) === -1)
}
- test( "andNot len(bitsetX) > len(bitsetY)" ) {
- val setBitsX = Seq( 0, 1, 3, 37, 38, 41, 85)
- val setBitsY = Seq( 0, 2, 3, 37, 41, 48 )
+ test("andNot len(bitsetX) > len(bitsetY)") {
+ val setBitsX = Seq(0, 1, 3, 37, 38, 41, 85)
+ val setBitsY = Seq(0, 2, 3, 37, 41, 48 )
val bitsetX = new BitSet(100)
- setBitsX.foreach( i => bitsetX.set(i))
+ setBitsX.foreach(i => bitsetX.set(i))
val bitsetY = new BitSet(60)
- setBitsY.foreach( i => bitsetY.set(i))
+ setBitsY.foreach(i => bitsetY.set(i))
- val bitsetDiff = bitsetX.andNot( bitsetY )
+ val bitsetDiff = bitsetX.andNot(bitsetY )
assert(bitsetDiff.nextSetBit(0) === 1)
assert(bitsetDiff.nextSetBit(1) === 1)
@@ -153,7 +153,7 @@ class BitSetSuite extends SparkFunSuite {
assert(bitsetDiff.nextSetBit(86) === -1)
}
- test( "[gs]etUntil" ) {
+ test("[gs]etUntil") {
val bitSet = new BitSet(100)
bitSet.setUntil(bitSet.capacity)
diff --git a/examples/src/main/scala/org/apache/spark/examples/LocalALS.scala b/examples/src/main/scala/org/apache/spark/examples/LocalALS.scala
index 97aefac025e55..b8e28e42e0a0e 100644
--- a/examples/src/main/scala/org/apache/spark/examples/LocalALS.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/LocalALS.scala
@@ -55,7 +55,7 @@ object LocalALS {
math.sqrt(sumSqs / (M.toDouble * U.toDouble))
}
- def updateMovie(i: Int, m: RealVector, us: Array[RealVector], R: RealMatrix) : RealVector = {
+ def updateMovie(i: Int, m: RealVector, us: Array[RealVector], R: RealMatrix): RealVector = {
var XtX: RealMatrix = new Array2DRowRealMatrix(F, F)
var Xty: RealVector = new ArrayRealVector(F)
// For each user that rated the movie
@@ -74,7 +74,7 @@ object LocalALS {
new CholeskyDecomposition(XtX).getSolver.solve(Xty)
}
- def updateUser(j: Int, u: RealVector, ms: Array[RealVector], R: RealMatrix) : RealVector = {
+ def updateUser(j: Int, u: RealVector, ms: Array[RealVector], R: RealMatrix): RealVector = {
var XtX: RealMatrix = new Array2DRowRealMatrix(F, F)
var Xty: RealVector = new ArrayRealVector(F)
// For each movie that the user rated
diff --git a/examples/src/main/scala/org/apache/spark/examples/SparkALS.scala b/examples/src/main/scala/org/apache/spark/examples/SparkALS.scala
index a99ddd9fd37db..f3bc9daf16280 100644
--- a/examples/src/main/scala/org/apache/spark/examples/SparkALS.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/SparkALS.scala
@@ -57,7 +57,7 @@ object SparkALS {
math.sqrt(sumSqs / (M.toDouble * U.toDouble))
}
- def update(i: Int, m: RealVector, us: Array[RealVector], R: RealMatrix) : RealVector = {
+ def update(i: Int, m: RealVector, us: Array[RealVector], R: RealMatrix): RealVector = {
val U = us.length
val F = us(0).getDimension
var XtX: RealMatrix = new Array2DRowRealMatrix(F, F)
diff --git a/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala b/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
index 8f8262db374b8..9f905ef66f38c 100644
--- a/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
@@ -46,7 +46,7 @@ object AggregateMessagesExample {
// Create a graph with "age" as the vertex property.
// Here we use a random graph for simplicity.
val graph: Graph[Double, Int] =
- GraphGenerators.logNormalGraph(sc, numVertices = 100).mapVertices( (id, _) => id.toDouble )
+ GraphGenerators.logNormalGraph(sc, numVertices = 100).mapVertices((id, _) => id.toDouble )
// Compute the number of older followers and their total age
val olderFollowers: VertexRDD[(Int, Double)] = graph.aggregateMessages[(Int, Double)](
triplet => { // Map Function
@@ -60,7 +60,7 @@ object AggregateMessagesExample {
)
// Divide total age by number of older followers to get average age of older followers
val avgAgeOfOlderFollowers: VertexRDD[Double] =
- olderFollowers.mapValues( (id, value) =>
+ olderFollowers.mapValues((id, value) =>
value match { case (count, totalAge) => totalAge / count } )
// Display the results
avgAgeOfOlderFollowers.collect.foreach(println(_))
diff --git a/examples/src/main/scala/org/apache/spark/examples/graphx/ComprehensiveExample.scala b/examples/src/main/scala/org/apache/spark/examples/graphx/ComprehensiveExample.scala
index 6598863bd2ea0..b0598d7500b2a 100644
--- a/examples/src/main/scala/org/apache/spark/examples/graphx/ComprehensiveExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/graphx/ComprehensiveExample.scala
@@ -47,7 +47,7 @@ object ComprehensiveExample {
// $example on$
// Load my user data and parse into tuples of user id and attribute list
val users = (sc.textFile("data/graphx/users.txt")
- .map(line => line.split(",")).map( parts => (parts.head.toLong, parts.tail) ))
+ .map(line => line.split(",")).map(parts => (parts.head.toLong, parts.tail)))
// Parse the edge data which is already in userId -> userId format
val followerGraph = GraphLoader.edgeListFile(sc, "data/graphx/followers.txt")
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/RandomRDDGeneration.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/RandomRDDGeneration.scala
index 7ccbb5a0640cd..4f1813b9f128e 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/RandomRDDGeneration.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/RandomRDDGeneration.scala
@@ -44,13 +44,13 @@ object RandomRDDGeneration {
println(s"Generated RDD of ${normalRDD.count()}" +
" examples sampled from the standard normal distribution")
println(" First 5 samples:")
- normalRDD.take(5).foreach( x => println(s" $x") )
+ normalRDD.take(5).foreach(x => println(s" $x") )
// Example: RandomRDDs.normalVectorRDD
val normalVectorRDD = RandomRDDs.normalVectorRDD(sc, numRows = numExamples, numCols = 2)
println(s"Generated RDD of ${normalVectorRDD.count()} examples of length-2 vectors.")
println(" First 5 samples:")
- normalVectorRDD.take(5).foreach( x => println(s" $x") )
+ normalVectorRDD.take(5).foreach(x => println(s" $x") )
println()
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/FlumeEventCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/FlumeEventCount.scala
index 91e52e4eff5a7..d4fb5f6b5d101 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/FlumeEventCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/FlumeEventCount.scala
@@ -61,7 +61,7 @@ object FlumeEventCount {
val stream = FlumeUtils.createStream(ssc, host, port, StorageLevel.MEMORY_ONLY_SER_2)
// Print out the count of events received from this server in each batch
- stream.count().map(cnt => "Received " + cnt + " flume events." ).print()
+ stream.count().map(cnt => "Received " + cnt + " flume events.").print()
ssc.start()
ssc.awaitTermination()
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/FlumePollingEventCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/FlumePollingEventCount.scala
index dd725d72c23ef..6780f638cade1 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/FlumePollingEventCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/FlumePollingEventCount.scala
@@ -58,7 +58,7 @@ object FlumePollingEventCount {
val stream = FlumeUtils.createPollingStream(ssc, host, port)
// Print out the count of events received from this server in each batch
- stream.count().map(cnt => "Received " + cnt + " flume events." ).print()
+ stream.count().map(cnt => "Received " + cnt + " flume events.").print()
ssc.start()
ssc.awaitTermination()
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
index 91893df4ec32f..eae4175583638 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
@@ -205,7 +205,7 @@ class KafkaRelationSuite extends QueryTest with BeforeAndAfter with SharedSQLCon
"for batch queries on Kafka")
// Now do it with an explicit json start offset indicating latest
- val startPartitionOffsets = Map( new TopicPartition("t", 0) -> -1L)
+ val startPartitionOffsets = Map(new TopicPartition("t", 0) -> -1L)
val startingOffsets = JsonUtils.partitionOffsets(startPartitionOffsets)
testBadOptions("subscribe" -> "t", "startingOffsets" -> startingOffsets)(
"startingOffsets for t-0 can't be latest for batch queries on Kafka")
diff --git a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala
index ef1968585be60..f96fe6666c749 100644
--- a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala
+++ b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala
@@ -188,7 +188,7 @@ private[kafka] class KafkaTestUtils extends Logging {
/** Send the array of messages to the Kafka broker */
def sendMessages(topic: String, messages: Array[String]): Unit = {
producer = new Producer[String, String](new ProducerConfig(producerConfiguration))
- producer.send(messages.map { new KeyedMessage[String, String](topic, _ ) }: _*)
+ producer.send(messages.map { new KeyedMessage[String, String](topic, _) }: _*)
producer.close()
producer = null
}
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala b/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala
index 475bccf9bfc76..a24c4ff6184e5 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/GraphOps.scala
@@ -397,7 +397,7 @@ class GraphOps[VD: ClassTag, ED: ClassTag](graph: Graph[VD, ED]) extends Seriali
* that all random walks are started relative to the source vertices
*/
def staticParallelPersonalizedPageRank(sources: Array[VertexId], numIter: Int,
- resetProb: Double = 0.15) : Graph[Vector, Double] = {
+ resetProb: Double = 0.15): Graph[Vector, Double] = {
PageRank.runParallelPersonalizedPageRank(graph, numIter, resetProb, sources)
}
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
index 5d2a53782b55d..0cbca72721a68 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
@@ -103,7 +103,7 @@ class GraphImpl[VD: ClassTag, ED: ClassTag] protected (
(part, (e.srcId, e.dstId, e.attr))
}
.partitionBy(new HashPartitioner(numPartitions))
- .mapPartitionsWithIndex( { (pid, iter) =>
+ .mapPartitionsWithIndex({ (pid, iter) =>
val builder = new EdgePartitionBuilder[ED, VD]()(edTag, vdTag)
iter.foreach { message =>
val data = message._2
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
index fd7b7f7c1c487..d9fe596d3e4a9 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
@@ -122,7 +122,7 @@ object PageRank extends Logging {
// Associate the degree with each vertex
.outerJoinVertices(graph.outDegrees) { (vid, vdata, deg) => deg.getOrElse(0) }
// Set the weight on the edges based on the degree
- .mapTriplets( e => 1.0 / e.srcAttr, TripletFields.Src )
+ .mapTriplets(e => 1.0 / e.srcAttr, TripletFields.Src)
// Set the vertex attributes to the initial pagerank values
.mapVertices { (id, attr) =>
if (!(id != src && personalized)) 1.0 else 0.0
@@ -227,7 +227,7 @@ object PageRank extends Logging {
// and adding start nodes back in with activation resetProb
val rankUpdates = rankGraph.aggregateMessages[BV[Double]](
ctx => ctx.sendToDst(ctx.srcAttr *:* ctx.attr),
- (a : BV[Double], b : BV[Double]) => a +:+ b, TripletFields.Src)
+ (a: BV[Double], b: BV[Double]) => a +:+ b, TripletFields.Src)
rankGraph = rankGraph.outerJoinVertices(rankUpdates) {
(vid, oldRank, msgSumOpt) =>
@@ -310,7 +310,7 @@ object PageRank extends Logging {
(vid, vdata, deg) => deg.getOrElse(0)
}
// Set the weight on the edges based on the degree
- .mapTriplets( e => 1.0 / e.srcAttr )
+ .mapTriplets(e => 1.0 / e.srcAttr)
// Set the vertex attributes to (initialPR, delta = 0)
.mapVertices { (id, attr) =>
if (id == src) (0.0, Double.NegativeInfinity) else (0.0, 0.0)
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/util/GraphGenerators.scala b/graphx/src/main/scala/org/apache/spark/graphx/util/GraphGenerators.scala
index 2b3e5f98c4fe5..079acf2d45737 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/util/GraphGenerators.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/util/GraphGenerators.scala
@@ -241,12 +241,12 @@ object GraphGenerators extends Logging {
def sub2ind(r: Int, c: Int): VertexId = r * cols + c
val vertices: RDD[(VertexId, (Int, Int))] = sc.parallelize(0 until rows).flatMap { r =>
- (0 until cols).map( c => (sub2ind(r, c), (r, c)) )
+ (0 until cols).map(c => (sub2ind(r, c), (r, c)))
}
val edges: RDD[Edge[Double]] =
vertices.flatMap{ case (vid, (r, c)) =>
- (if (r + 1 < rows) { Seq( (sub2ind(r, c), sub2ind(r + 1, c))) } else { Seq.empty }) ++
- (if (c + 1 < cols) { Seq( (sub2ind(r, c), sub2ind(r, c + 1))) } else { Seq.empty })
+ (if (r + 1 < rows) { Seq((sub2ind(r, c), sub2ind(r + 1, c))) } else { Seq.empty }) ++
+ (if (c + 1 < cols) { Seq((sub2ind(r, c), sub2ind(r, c + 1))) } else { Seq.empty })
}.map{ case (src, dst) => Edge(src, dst, 1.0) }
Graph(vertices, edges)
} // end of gridGraph
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/GraphSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/GraphSuite.scala
index 88b59a343a83c..3aaca3a40b26f 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/GraphSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/GraphSuite.scala
@@ -59,9 +59,9 @@ class GraphSuite extends SparkFunSuite with LocalSparkContext {
val edges: RDD[Edge[Int]] = sc.parallelize(rawEdges).map { case (s, t) => Edge(s, t, 1) }
val vertices: RDD[(VertexId, Boolean)] = sc.parallelize((0L until 10L).map(id => (id, true)))
val graph = Graph(vertices, edges, false)
- assert( graph.edges.count() === rawEdges.size )
+ assert(graph.edges.count() === rawEdges.size )
// Vertices not explicitly provided but referenced by edges should be created automatically
- assert( graph.vertices.count() === 100)
+ assert(graph.vertices.count() === 100)
graph.triplets.collect().foreach { et =>
assert((et.srcId < 10 && et.srcAttr) || (et.srcId >= 10 && !et.srcAttr))
assert((et.dstId < 10 && et.dstAttr) || (et.dstId >= 10 && !et.dstAttr))
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/ConnectedComponentsSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/ConnectedComponentsSuite.scala
index 1b81423563372..8b485e3a3b64b 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/ConnectedComponentsSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/ConnectedComponentsSuite.scala
@@ -53,7 +53,7 @@ class ConnectedComponentsSuite extends SparkFunSuite with LocalSparkContext {
val twoChains = Graph.fromEdgeTuples(rawEdges, 1.0)
val ccGraph = twoChains.connectedComponents()
val vertices = ccGraph.vertices.collect()
- for ( (id, cc) <- vertices ) {
+ for ((id, cc) <- vertices) {
if (id < 10) {
assert(cc === 0)
} else {
@@ -79,7 +79,7 @@ class ConnectedComponentsSuite extends SparkFunSuite with LocalSparkContext {
val twoChains = Graph.fromEdgeTuples(rawEdges, true).reverse
val ccGraph = twoChains.connectedComponents()
val vertices = ccGraph.vertices.collect()
- for ( (id, cc) <- vertices ) {
+ for ((id, cc) <- vertices) {
if (id < 10) {
assert(cc === 0)
} else {
@@ -87,7 +87,7 @@ class ConnectedComponentsSuite extends SparkFunSuite with LocalSparkContext {
}
}
val ccMap = vertices.toMap
- for ( id <- 0 until 20 ) {
+ for (id <- 0 until 20) {
if (id < 10) {
assert(ccMap(id) === 0)
} else {
@@ -121,7 +121,7 @@ class ConnectedComponentsSuite extends SparkFunSuite with LocalSparkContext {
val graph = Graph(users, relationships, defaultUser)
val ccGraph = graph.connectedComponents()
val vertices = ccGraph.vertices.collect()
- for ( (id, cc) <- vertices ) {
+ for ((id, cc) <- vertices) {
assert(cc === 0)
}
}
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
index 9779553ce85d1..7854dff2ee926 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
@@ -47,7 +47,7 @@ object GridPageRank {
pr = new Array[Double](nRows * nCols)
for (ind <- 0 until (nRows * nCols)) {
pr(ind) = resetProb + (1.0 - resetProb) *
- inNbrs(ind).map( nbr => oldPr(nbr) / outDegree(nbr)).sum
+ inNbrs(ind).map(nbr => oldPr(nbr) / outDegree(nbr)).sum
}
}
val prSum = pr.sum
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/TriangleCountSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/TriangleCountSuite.scala
index f19c3acdc85cf..feeebf0171847 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/TriangleCountSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/TriangleCountSuite.scala
@@ -26,7 +26,7 @@ class TriangleCountSuite extends SparkFunSuite with LocalSparkContext {
test("Count a single triangle") {
withSpark { sc =>
- val rawEdges = sc.parallelize(Array( 0L -> 1L, 1L -> 2L, 2L -> 0L ), 2)
+ val rawEdges = sc.parallelize(Array(0L -> 1L, 1L -> 2L, 2L -> 0L), 2)
val graph = Graph.fromEdgeTuples(rawEdges, true).cache()
val triangleCount = graph.triangleCount()
val verts = triangleCount.vertices
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
index 3fbc0958a0f11..3fe0d7c391f24 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
@@ -435,7 +435,7 @@ object Vectors {
* A dense vector represented by a value array.
*/
@Since("2.0.0")
-class DenseVector @Since("2.0.0") ( @Since("2.0.0") val values: Array[Double]) extends Vector {
+class DenseVector @Since("2.0.0") (@Since("2.0.0") val values: Array[Double]) extends Vector {
override def size: Int = values.length
diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala
index 3aea568cd6527..0f1bf6e9d8818 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala
@@ -119,6 +119,6 @@ private[ann] class SoftmaxLayerModelWithCrossEntropyLoss extends LayerModel with
override def loss(output: BDM[Double], target: BDM[Double], delta: BDM[Double]): Double = {
ApplyInPlace(output, target, delta, (o: Double, t: Double) => o - t)
- -Bsum( target *:* brzlog(output)) / output.cols
+ -Bsum(target *:* brzlog(output)) / output.cols
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
index 4b44878784c90..08e4376cb0fc1 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
@@ -507,7 +507,7 @@ private class VectorAttributeRewriter(
}
val otherCols = dataset.columns.filter(_ != vectorCol).map(dataset.col)
val rewrittenCol = dataset.col(vectorCol).as(vectorCol, metadata)
- dataset.select(otherCols :+ rewrittenCol : _*)
+ dataset.select(otherCols:+ rewrittenCol: _*)
}
override def transformSchema(schema: StructType): StructType = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala
index 21531eb057ad3..a6cedc961473f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala
@@ -49,7 +49,7 @@ private[r] object KSTestWrapper {
case Row(feature: Double) => feature
}
- val ksTestResult = kolmogorovSmirnovTest(rddData, distName, distParams : _*)
+ val ksTestResult = kolmogorovSmirnovTest(rddData, distName, distParams: _*)
new KSTestWrapper(ksTestResult, distName, distParams)
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 0d6e9034e5ce4..135095ddd4e4c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -198,7 +198,7 @@ private[ml] object TreeEnsembleModel {
* If -1, then numFeatures is set based on the max feature index in all trees.
* @return Feature importance values, of length numFeatures.
*/
- def featureImportances[M <: DecisionTreeModel : ClassTag](tree: M, numFeatures: Int): Vector = {
+ def featureImportances[M <: DecisionTreeModel: ClassTag](tree: M, numFeatures: Int): Vector = {
featureImportances(Array(tree), numFeatures)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
index b32d3f252ae59..c443c18343912 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
@@ -590,7 +590,7 @@ private[python] class PythonMLLibAPI extends Serializable {
data: JavaRDD[java.util.ArrayList[java.util.ArrayList[Any]]],
minSupport: Double,
maxPatternLength: Int,
- localProjDBSize: Int ): PrefixSpanModelWrapper = {
+ localProjDBSize: Int): PrefixSpanModelWrapper = {
val prefixSpan = new PrefixSpan()
.setMinSupport(minSupport)
.setMaxPatternLength(maxPatternLength)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala
index b2437b845f826..8e543a995ea63 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala
@@ -270,9 +270,11 @@ object PowerIterationClustering extends Logging {
Graph(vD, graph.edges)
.mapTriplets(
e => e.attr / math.max(e.srcAttr, MLUtils.EPSILON),
+ // scalastyle:off
new TripletFields(/* useSrc */ true,
/* useDst */ false,
/* useEdge */ true))
+ // scalastyle:on
}
/**
@@ -301,9 +303,11 @@ object PowerIterationClustering extends Logging {
Graph(vD, gA.edges)
.mapTriplets(
e => e.attr / math.max(e.srcAttr, MLUtils.EPSILON),
+ // scalastyle:off
new TripletFields(/* useSrc */ true,
- /* useDst */ false,
- /* useEdge */ true))
+ /* useDst */ false,
+ /* useEdge */ true))
+ // scalastyle:on
}
/**
@@ -364,9 +368,11 @@ object PowerIterationClustering extends Logging {
val v = curG.aggregateMessages[Double](
sendMsg = ctx => ctx.sendToSrc(ctx.attr * ctx.dstAttr),
mergeMsg = _ + _,
+ // scalastyle:off
new TripletFields(/* useSrc */ false,
/* useDst */ true,
/* useEdge */ true)).cache()
+ // scalastyle:on
// normalize v
val norm = v.values.map(math.abs).sum()
logInfo(s"$msgPrefix: norm(v) = $norm.")
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
index 3a2be236f1257..2c4747faeb94a 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
@@ -333,7 +333,7 @@ object NaiveBayesSuite {
val y = calcLabel(rnd.nextDouble(), _pi)
val xi = modelType match {
case Bernoulli => Array.tabulate[Double] (D) { j =>
- if (rnd.nextDouble () < _theta(y)(j) ) 1 else 0
+ if (rnd.nextDouble () < _theta(y)(j)) 1 else 0
}
case Multinomial =>
val mult = BrzMultinomial(BDV(_theta(y)))
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
index 08b800b7e4183..60aff9f829695 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
@@ -274,10 +274,10 @@ object GaussianMixtureSuite extends SparkFunSuite {
val denseData = Seq(
Vectors.dense(-5.1971), Vectors.dense(-2.5359), Vectors.dense(-3.8220),
- Vectors.dense(-5.2211), Vectors.dense(-5.0602), Vectors.dense( 4.7118),
- Vectors.dense( 6.8989), Vectors.dense( 3.4592), Vectors.dense( 4.6322),
- Vectors.dense( 5.7048), Vectors.dense( 4.6567), Vectors.dense( 5.5026),
- Vectors.dense( 4.5605), Vectors.dense( 5.2043), Vectors.dense( 6.2734)
+ Vectors.dense(-5.2211), Vectors.dense(-5.0602), Vectors.dense(4.7118),
+ Vectors.dense(6.8989), Vectors.dense(3.4592), Vectors.dense(4.6322),
+ Vectors.dense(5.7048), Vectors.dense(4.6567), Vectors.dense(5.5026),
+ Vectors.dense(4.5605), Vectors.dense(5.2043), Vectors.dense(6.2734)
)
val decompositionData: Seq[Vector] = Seq.tabulate(25) { i: Int =>
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala
index ee2ba73fa96d5..c9867afb62de2 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/ImputerSuite.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.{DataFrame, Row}
class ImputerSuite extends SparkFunSuite with MLlibTestSparkContext with DefaultReadWriteTest {
test("Imputer for Double with default missing Value NaN") {
- val df = spark.createDataFrame( Seq(
+ val df = spark.createDataFrame(Seq(
(0, 1.0, 4.0, 1.0, 1.0, 4.0, 4.0),
(1, 11.0, 12.0, 11.0, 11.0, 12.0, 12.0),
(2, 3.0, Double.NaN, 3.0, 3.0, 10.0, 12.0),
@@ -39,7 +39,7 @@ class ImputerSuite extends SparkFunSuite with MLlibTestSparkContext with Default
}
test("Imputer should handle NaNs when computing surrogate value, if missingValue is not NaN") {
- val df = spark.createDataFrame( Seq(
+ val df = spark.createDataFrame(Seq(
(0, 1.0, 1.0, 1.0),
(1, 3.0, 3.0, 3.0),
(2, Double.NaN, Double.NaN, Double.NaN),
@@ -51,7 +51,7 @@ class ImputerSuite extends SparkFunSuite with MLlibTestSparkContext with Default
}
test("Imputer for Float with missing Value -1.0") {
- val df = spark.createDataFrame( Seq(
+ val df = spark.createDataFrame(Seq(
(0, 1.0F, 1.0F, 1.0F),
(1, 3.0F, 3.0F, 3.0F),
(2, 10.0F, 10.0F, 10.0F),
@@ -64,7 +64,7 @@ class ImputerSuite extends SparkFunSuite with MLlibTestSparkContext with Default
}
test("Imputer should impute null as well as 'missingValue'") {
- val rawDf = spark.createDataFrame( Seq(
+ val rawDf = spark.createDataFrame(Seq(
(0, 4.0, 4.0, 4.0),
(1, 10.0, 10.0, 10.0),
(2, 10.0, 10.0, 10.0),
@@ -77,7 +77,7 @@ class ImputerSuite extends SparkFunSuite with MLlibTestSparkContext with Default
}
test("Imputer throws exception when surrogate cannot be computed") {
- val df = spark.createDataFrame( Seq(
+ val df = spark.createDataFrame(Seq(
(0, Double.NaN, 1.0, 1.0),
(1, Double.NaN, 3.0, 3.0),
(2, Double.NaN, Double.NaN, Double.NaN)
@@ -95,7 +95,7 @@ class ImputerSuite extends SparkFunSuite with MLlibTestSparkContext with Default
}
test("Imputer input & output column validation") {
- val df = spark.createDataFrame( Seq(
+ val df = spark.createDataFrame(Seq(
(0, 1.0, 1.0, 1.0),
(1, Double.NaN, 3.0, 3.0),
(2, Double.NaN, Double.NaN, Double.NaN)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
index f219f775b2186..886990ed2e7dd 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
@@ -45,7 +45,7 @@ class QuantileDiscretizerSuite
val relativeError = discretizer.getRelativeError
val isGoodBucket = udf {
- (size: Int) => math.abs( size - (datasetSize / numBuckets)) <= (relativeError * datasetSize)
+ (size: Int) => math.abs(size - (datasetSize / numBuckets)) <= (relativeError * datasetSize)
}
val numGoodBuckets = result.groupBy("result").count.filter(isGoodBucket($"count")).count
assert(numGoodBuckets === numBuckets,
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
index 5ec4c15387e94..37ef7e01a67db 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
@@ -60,7 +60,7 @@ object NaiveBayesSuite {
val y = calcLabel(rnd.nextDouble(), _pi)
val xi = modelType match {
case Bernoulli => Array.tabulate[Double] (D) { j =>
- if (rnd.nextDouble () < _theta(y)(j) ) 1 else 0
+ if (rnd.nextDouble () < _theta(y)(j)) 1 else 0
}
case Multinomial =>
val mult = BrzMultinomial(BDV(_theta(y)))
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/GaussianMixtureSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/GaussianMixtureSuite.scala
index 11189d8bd4776..36c291d81a4dc 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/GaussianMixtureSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/GaussianMixtureSuite.scala
@@ -190,10 +190,10 @@ class GaussianMixtureSuite extends SparkFunSuite with MLlibTestSparkContext {
val data = Array(
Vectors.dense(-5.1971), Vectors.dense(-2.5359), Vectors.dense(-3.8220),
- Vectors.dense(-5.2211), Vectors.dense(-5.0602), Vectors.dense( 4.7118),
- Vectors.dense( 6.8989), Vectors.dense( 3.4592), Vectors.dense( 4.6322),
- Vectors.dense( 5.7048), Vectors.dense( 4.6567), Vectors.dense( 5.5026),
- Vectors.dense( 4.5605), Vectors.dense( 5.2043), Vectors.dense( 6.2734)
+ Vectors.dense(-5.2211), Vectors.dense(-5.0602), Vectors.dense(4.7118),
+ Vectors.dense(6.8989), Vectors.dense(3.4592), Vectors.dense(4.6322),
+ Vectors.dense(5.7048), Vectors.dense(4.6567), Vectors.dense(5.5026),
+ Vectors.dense(4.5605), Vectors.dense(5.2043), Vectors.dense(6.2734)
)
val data2: Array[Vector] = Array.tabulate(25) { i: Int =>
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
index a660492c7ae59..8a09b1f441bd3 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
@@ -74,7 +74,7 @@ class MultilabelMetricsSuite extends SparkFunSuite with MLlibTestSparkContext {
val macroRecallDoc = 1.0 / 7 *
(1.0 / 2 + 1.0 / 2 + 0 / 1 + 1.0 / 1 + 2.0 / 2 + 2.0 / 2 + 1.0 / 2)
val macroF1MeasureDoc = (1.0 / 7) *
- 2 * ( 1.0 / (2 + 2) + 1.0 / (2 + 2) + 0 + 1.0 / (1 + 1) +
+ 2 * (1.0 / (2 + 2) + 1.0 / (2 + 2) + 0 + 1.0 / (1 + 1) +
2.0 / (2 + 2) + 2.0 / (3 + 2) + 1.0 / (1 + 2) )
val hammingLoss = (1.0 / (7 * 3)) * (2 + 2 + 1 + 0 + 0 + 1 + 1)
val strictAccuracy = 2.0 / 7
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
index 062ed1f93fa52..2737688267451 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
@@ -447,11 +447,11 @@ trait MesosSchedulerUtils extends Logging {
* @return the ono-zero values of the ports
*/
def nonZeroPortValuesFromConfig(conf: SparkConf): List[Long] = {
- managedPortNames.map(conf.getLong(_, 0)).filter( _ != 0)
+ managedPortNames.map(conf.getLong(_, 0)).filter(_ != 0)
}
/** Creates a mesos resource for a specific port number. */
- private def createResourcesFromPorts(portsAndRoles: List[(Long, String)]) : List[Resource] = {
+ private def createResourcesFromPorts(portsAndRoles: List[(Long, String)]): List[Resource] = {
portsAndRoles.flatMap{ case (port, role) =>
createMesosPortResource(List((port, port)), Some(role))}
}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
index 97b0e8aca3330..112cd938267f0 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
@@ -245,7 +245,7 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
handler.handleAllocatedContainers(Array(container1, container2))
handler.requestTotalExecutorsWithPreferredLocalities(1, 0, Map.empty, Set.empty)
- handler.executorIdToContainer.keys.foreach { id => handler.killExecutor(id ) }
+ handler.executorIdToContainer.keys.foreach { id => handler.killExecutor(id) }
val statuses = Seq(container1, container2).map { c =>
ContainerStatus.newInstance(c.getId(), ContainerState.COMPLETE, "Finished", 0)
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index 0a4073b03957c..8dc541db1933a 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -243,10 +243,26 @@ This file is divided into 3 sections:
-
+
- COMMA
+ COLON, COMMA, RPAREN
+
+
+
+
+
+ LPAREN
+
+
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
index 0b95a8821b05a..de04ced831002 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
@@ -272,7 +272,7 @@ object Encoders {
* An encoder for Scala's product type (tuples, case classes, etc).
* @since 2.0.0
*/
- def product[T <: Product : TypeTag]: Encoder[T] = ExpressionEncoder()
+ def product[T <: Product: TypeTag]: Encoder[T] = ExpressionEncoder()
/**
* An encoder for Scala's primitive int type.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index d580cf4d3391c..b32f99a6cbf7a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -60,7 +60,7 @@ object ScalaReflection extends ScalaReflection {
* Unlike `schemaFor`, this function doesn't do any massaging of types into the Spark SQL type
* system. As a result, ObjectType will be returned for things like boxed Integers
*/
- def dataTypeFor[T : TypeTag]: DataType = dataTypeFor(localTypeOf[T])
+ def dataTypeFor[T: TypeTag]: DataType = dataTypeFor(localTypeOf[T])
private def dataTypeFor(tpe: `Type`): DataType = ScalaReflectionLock.synchronized {
tpe match {
@@ -131,7 +131,7 @@ object ScalaReflection extends ScalaReflection {
* from ordinal 0 (since there are no names to map to). The actual location can be moved by
* calling resolve/bind with a new schema.
*/
- def deserializerFor[T : TypeTag]: Expression = {
+ def deserializerFor[T: TypeTag]: Expression = {
val tpe = localTypeOf[T]
val clsName = getClassNameFromType(tpe)
val walkedTypePath = s"""- root class: "$clsName"""" :: Nil
@@ -414,7 +414,7 @@ object ScalaReflection extends ScalaReflection {
* * the element type of [[Array]] or [[Seq]]: `array element class: "abc.xyz.MyClass"`
* * the field of [[Product]]: `field (class: "abc.xyz.MyClass", name: "myField")`
*/
- def serializerFor[T : TypeTag](inputObject: Expression): CreateNamedStruct = {
+ def serializerFor[T: TypeTag](inputObject: Expression): CreateNamedStruct = {
val tpe = localTypeOf[T]
val clsName = getClassNameFromType(tpe)
val walkedTypePath = s"""- root class: "$clsName"""" :: Nil
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 53536496d0457..11d80f64b34ac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -151,7 +151,7 @@ class Analyzer(
ResolveInlineTables(conf) ::
ResolveTimeZone(conf) ::
TypeCoercion.typeCoercionRules ++
- extendedResolutionRules : _*),
+ extendedResolutionRules: _*),
Batch("Post-Hoc Resolution", Once, postHocResolutionRules: _*),
Batch("View", Once,
AliasViewChild(conf)),
@@ -182,7 +182,7 @@ class Analyzer(
def substituteCTE(plan: LogicalPlan, cteRelations: Seq[(String, LogicalPlan)]): LogicalPlan = {
plan transformDown {
- case u : UnresolvedRelation =>
+ case u: UnresolvedRelation =>
cteRelations.find(x => resolver(x._1, u.tableIdentifier.table))
.map(_._2).getOrElse(u)
case other =>
@@ -917,7 +917,7 @@ class Analyzer(
})
case c: CreateNamedStruct if containsStar(c.valExprs) =>
val newChildren = c.children.grouped(2).flatMap {
- case Seq(k, s : Star) => CreateStruct(s.expand(child, resolver)).children
+ case Seq(k, s: Star) => CreateStruct(s.expand(child, resolver)).children
case kv => kv
}
c.copy(children = newChildren.toList )
@@ -2061,7 +2061,7 @@ class Analyzer(
joinedCols ++
lUniqueOutput.map(_.withNullability(true)) ++
rUniqueOutput.map(_.withNullability(true))
- case _ : InnerLike =>
+ case _: InnerLike =>
leftKeys ++ lUniqueOutput ++ rUniqueOutput
case _ =>
sys.error("Unsupported natural join type " + joinType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index f4b3e86052d8e..35f9f0137af68 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -525,7 +525,7 @@ object FunctionRegistry {
val f = constructors.find(_.getParameterTypes.toSeq == params).getOrElse {
throw new AnalysisException(s"Invalid number of arguments for function $name")
}
- Try(f.newInstance(expressions : _*).asInstanceOf[Expression]) match {
+ Try(f.newInstance(expressions: _*).asInstanceOf[Expression]) match {
case Success(e) => e
case Failure(e) =>
// the exception is an invocation exception. To get a meaningful message, we need the
@@ -561,7 +561,7 @@ object FunctionRegistry {
/**
* Creates an [[ExpressionInfo]] for the function as defined by expression T using the given name.
*/
- private def expressionInfo[T <: Expression : ClassTag](name: String): ExpressionInfo = {
+ private def expressionInfo[T <: Expression: ClassTag](name: String): ExpressionInfo = {
val clazz = scala.reflect.classTag[T].runtimeClass
val df = clazz.getAnnotation(classOf[ExpressionDescription])
if (df != null) {
@@ -571,7 +571,7 @@ object FunctionRegistry {
}
}
- private def expressionGeneratorOuter[T <: Generator : ClassTag](name: String)
+ private def expressionGeneratorOuter[T <: Generator: ClassTag](name: String)
: (String, (ExpressionInfo, FunctionBuilder)) = {
val (_, (info, generatorBuilder)) = expression[T](name)
val outerBuilder = (args: Seq[Expression]) => {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index f6792569b704e..4291f368bcde1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -63,10 +63,11 @@ package object dsl {
trait ImplicitOperators {
def expr: Expression
+ // scalastyle:off
def unary_- : Expression = UnaryMinus(expr)
def unary_! : Predicate = Not(expr)
def unary_~ : Expression = BitwiseNot(expr)
-
+ // scalastyle:on
def + (other: Expression): Expression = Add(expr, other)
def - (other: Expression): Expression = Subtract(expr, other)
def * (other: Expression): Expression = Multiply(expr, other)
@@ -144,7 +145,7 @@ package object dsl {
// Note that if we make ExpressionConversions an object rather than a trait, we can
// then make this a value class to avoid the small penalty of runtime instantiation.
def $(args: Any*): analysis.UnresolvedAttribute = {
- analysis.UnresolvedAttribute(sc.s(args : _*))
+ analysis.UnresolvedAttribute(sc.s(args: _*))
}
}
@@ -298,11 +299,11 @@ package object dsl {
def where(condition: Expression): LogicalPlan = Filter(condition, logicalPlan)
- def filter[T : Encoder](func: T => Boolean): LogicalPlan = TypedFilter(func, logicalPlan)
+ def filter[T: Encoder](func: T => Boolean): LogicalPlan = TypedFilter(func, logicalPlan)
- def serialize[T : Encoder]: LogicalPlan = CatalystSerde.serialize[T](logicalPlan)
+ def serialize[T: Encoder]: LogicalPlan = CatalystSerde.serialize[T](logicalPlan)
- def deserialize[T : Encoder]: LogicalPlan = CatalystSerde.deserialize[T](logicalPlan)
+ def deserialize[T: Encoder]: LogicalPlan = CatalystSerde.deserialize[T](logicalPlan)
def limit(limitExpr: Expression): LogicalPlan = Limit(limitExpr, logicalPlan)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index efc2882f0a3d3..22cbb8189f5f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -43,7 +43,7 @@ import org.apache.spark.util.Utils
* to the name `value`.
*/
object ExpressionEncoder {
- def apply[T : TypeTag](): ExpressionEncoder[T] = {
+ def apply[T: TypeTag](): ExpressionEncoder[T] = {
// We convert the not-serializable TypeTag into StructType and ClassTag.
val mirror = ScalaReflection.mirror
val tpe = typeTag[T].in(mirror).tpe
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala
index 59f7969e56144..ce9ebf98bf5e3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/package.scala
@@ -26,7 +26,7 @@ package object encoders {
* references from a specific schema.) This requirement allows us to preserve whether a given
* object type is being bound by name or by ordinal when doing resolution.
*/
- def encoderFor[A : Encoder]: ExpressionEncoder[A] = implicitly[Encoder[A]] match {
+ def encoderFor[A: Encoder]: ExpressionEncoder[A] = implicitly[Encoder[A]] match {
case e: ExpressionEncoder[A] =>
e.assertUnresolved()
e
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
index 4859e0c537610..ee0a22123a3e3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
@@ -92,7 +92,7 @@ case class CallMethodViaReflection(children: Seq[Expression])
}
i += 1
}
- val ret = method.invoke(null, buffer : _*)
+ val ret = method.invoke(null, buffer: _*)
UTF8String.fromString(String.valueOf(ret))
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
index 8433a93ea3032..549ce395cefb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
@@ -58,7 +58,7 @@ import org.apache.spark.SparkException
case class Percentile(
child: Expression,
percentageExpression: Expression,
- frequencyExpression : Expression,
+ frequencyExpression: Expression,
mutableAggBufferOffset: Int = 0,
inputAggBufferOffset: Int = 0)
extends TypedImperativeAggregate[OpenHashMap[AnyRef, Long]] with ImplicitCastInputTypes {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index eaeaf08c37b4e..d079385e1eb1f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -155,7 +155,7 @@ object Literal {
Literal(CatalystTypeConverters.convertToCatalyst(v), dataType)
}
- def create[T : TypeTag](v: T): Literal = Try {
+ def create[T: TypeTag](v: T): Literal = Try {
val ScalaReflection.Schema(dataType, _) = ScalaReflection.schemaFor[T]
val convert = CatalystTypeConverters.createToCatalystConverter(dataType)
Literal(convert(v), dataType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index 3fc4bb7041636..3d9de69318220 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -67,7 +67,7 @@ case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCa
private val errMsg = s"'${child.simpleString}' is not true!"
- override def eval(input: InternalRow) : Any = {
+ override def eval(input: InternalRow): Any = {
val v = child.eval(input)
if (v == null || java.lang.Boolean.FALSE.equals(v)) {
throw new RuntimeException(errMsg)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 4b651836ff4d2..fa2b10e2d8235 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -182,7 +182,7 @@ case class Invoke(
dataType: DataType,
arguments: Seq[Expression] = Nil,
propagateNull: Boolean = true,
- returnNullable : Boolean = true) extends InvokeLike {
+ returnNullable: Boolean = true) extends InvokeLike {
override def nullable: Boolean = targetObject.nullable || needNullCheck || returnNullable
override def children: Seq[Expression] = targetObject +: arguments
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 83fdcfce9c3bd..fd2e06d9fb6fc 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -57,7 +57,7 @@ case class Concat(children: Seq[Expression]) extends Expression with ImplicitCas
override def eval(input: InternalRow): Any = {
val inputs = children.map(_.eval(input).asInstanceOf[UTF8String])
- UTF8String.concat(inputs : _*)
+ UTF8String.concat(inputs: _*)
}
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -117,7 +117,7 @@ case class ConcatWs(children: Seq[Expression])
case null => Iterator(null.asInstanceOf[UTF8String])
}
}
- UTF8String.concatWs(flatInputs.head, flatInputs.tail : _*)
+ UTF8String.concatWs(flatInputs.head, flatInputs.tail: _*)
}
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
@@ -1640,7 +1640,7 @@ case class FormatNumber(x: Expression, d: Expression)
def typeHelper(p: String): String = {
x.dataType match {
- case _ : DecimalType => s"""$p.toJavaBigDecimal()"""
+ case _: DecimalType => s"""$p.toJavaBigDecimal()"""
case _ => s"$p"
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
index 3a7543e2141e9..73e0d12c25576 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
@@ -433,7 +433,7 @@ object JoinReorderDPFilters extends PredicateHelper {
def starJoinFilter(
oneSideJoinPlan: Set[Int],
otherSideJoinPlan: Set[Int],
- filters: JoinGraphInfo) : Boolean = {
+ filters: JoinGraphInfo): Boolean = {
val starJoins = filters.starJoins
val nonStarJoins = filters.nonStarJoins
val join = oneSideJoinPlan.union(otherSideJoinPlan)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index 66b8ca62e5e4c..a97575ff1856f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -361,7 +361,7 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper {
// a branc with a TRue condition eliminates all following branches,
// these branches can be pruned away
val (h, t) = branches.span(_._1 != TrueLiteral)
- CaseWhen( h :+ t.head, None)
+ CaseWhen(h :+ t.head, None)
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
index 2a3e07aebe709..d3f0e796098fd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/subquery.scala
@@ -44,7 +44,7 @@ import org.apache.spark.sql.types._
object RewritePredicateSubquery extends Rule[LogicalPlan] with PredicateHelper {
private def getValueExpression(e: Expression): Seq[Expression] = {
e match {
- case cns : CreateNamedStruct => cns.valExprs
+ case cns: CreateNamedStruct => cns.valExprs
case expr => Seq(expr)
}
}
@@ -268,7 +268,7 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
* Statically evaluate an expression containing zero or more placeholders, given a set
* of bindings for placeholder values.
*/
- private def evalExpr(expr: Expression, bindings: Map[ExprId, Option[Any]]) : Option[Any] = {
+ private def evalExpr(expr: Expression, bindings: Map[ExprId, Option[Any]]): Option[Any] = {
val rewrittenExpr = expr transform {
case r: AttributeReference =>
bindings(r.exprId) match {
@@ -282,7 +282,7 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
/**
* Statically evaluate an expression containing one or more aggregates on an empty input.
*/
- private def evalAggOnZeroTups(expr: Expression) : Option[Any] = {
+ private def evalAggOnZeroTups(expr: Expression): Option[Any] = {
// AggregateExpressions are Unevaluable, so we need to replace all aggregates
// in the expression with the value they would return for zero input tuples.
// Also replace attribute refs (for example, for grouping columns) with NULL.
@@ -302,11 +302,11 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
* [[org.apache.spark.sql.catalyst.analysis.CheckAnalysis]]. If the checks in
* CheckAnalysis become less restrictive, this method will need to change.
*/
- private def evalSubqueryOnZeroTups(plan: LogicalPlan) : Option[Any] = {
+ private def evalSubqueryOnZeroTups(plan: LogicalPlan): Option[Any] = {
// Inputs to this method will start with a chain of zero or more SubqueryAlias
// and Project operators, followed by an optional Filter, followed by an
// Aggregate. Traverse the operators recursively.
- def evalPlan(lp : LogicalPlan) : Map[ExprId, Option[Any]] = lp match {
+ def evalPlan(lp: LogicalPlan): Map[ExprId, Option[Any]] = lp match {
case SubqueryAlias(_, child) => evalPlan(child)
case Filter(condition, child) =>
val bindings = evalPlan(child)
@@ -349,7 +349,7 @@ object RewriteCorrelatedScalarSubquery extends Rule[LogicalPlan] {
* (first part of returned value), the HAVING clause of the innermost query block
* (optional second part) and the parts below the HAVING CLAUSE (third part).
*/
- private def splitSubquery(plan: LogicalPlan) : (Seq[LogicalPlan], Option[Filter], Aggregate) = {
+ private def splitSubquery(plan: LogicalPlan): (Seq[LogicalPlan], Option[Filter], Aggregate) = {
val topPart = ArrayBuffer.empty[LogicalPlan]
var bottomPart: LogicalPlan = plan
while (true) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala
index 60d7361242c69..98f6faede8537 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/LegacyTypeStringParser.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.types._
object LegacyTypeStringParser extends RegexParsers {
protected lazy val primitiveType: Parser[DataType] =
- ( "StringType" ^^^ StringType
+ ("StringType" ^^^ StringType
| "FloatType" ^^^ FloatType
| "IntegerType" ^^^ IntegerType
| "ByteType" ^^^ ByteType
@@ -65,7 +65,7 @@ object LegacyTypeStringParser extends RegexParsers {
}
protected lazy val boolVal: Parser[Boolean] =
- ( "true" ^^^ true
+ ("true" ^^^ true
| "false" ^^^ false
)
@@ -75,7 +75,7 @@ object LegacyTypeStringParser extends RegexParsers {
}
protected lazy val dataType: Parser[DataType] =
- ( arrayType
+ (arrayType
| mapType
| structType
| primitiveType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index e89caabf252d7..7dd54db1000a9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -51,7 +51,7 @@ case class Project(projectList: Seq[NamedExpression], child: LogicalPlan) extend
override def maxRows: Option[Long] = child.maxRows
override lazy val resolved: Boolean = {
- val hasSpecialExpressions = projectList.exists ( _.collect {
+ val hasSpecialExpressions = projectList.exists (_.collect {
case agg: AggregateExpression => agg
case generator: Generator => generator
case window: WindowExpression => window
@@ -237,7 +237,7 @@ case class Union(children: Seq[LogicalPlan]) extends LogicalPlan {
override lazy val resolved: Boolean = {
// allChildrenCompatible needs to be evaluated after childrenResolved
def allChildrenCompatible: Boolean =
- children.tail.forall( child =>
+ children.tail.forall(child =>
// compare the attribute number with the first child
child.output.length == children.head.output.length &&
// compare the data types with the first child
@@ -524,7 +524,7 @@ case class Range(
override def computeStats: Statistics = {
val sizeInBytes = LongType.defaultSize * numElements
- Statistics( sizeInBytes = sizeInBytes )
+ Statistics(sizeInBytes = sizeInBytes)
}
override def simpleString: String = {
@@ -539,7 +539,7 @@ case class Aggregate(
extends UnaryNode {
override lazy val resolved: Boolean = {
- val hasWindowExpressions = aggregateExpressions.exists ( _.collect {
+ val hasWindowExpressions = aggregateExpressions.exists (_.collect {
case window: WindowExpression => window
}.nonEmpty
)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
index bfb70c2ef4c89..5aa6b70f2e8a2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
@@ -31,16 +31,16 @@ import org.apache.spark.sql.types._
import org.apache.spark.util.Utils
object CatalystSerde {
- def deserialize[T : Encoder](child: LogicalPlan): DeserializeToObject = {
+ def deserialize[T: Encoder](child: LogicalPlan): DeserializeToObject = {
val deserializer = UnresolvedDeserializer(encoderFor[T].deserializer)
DeserializeToObject(deserializer, generateObjAttr[T], child)
}
- def serialize[T : Encoder](child: LogicalPlan): SerializeFromObject = {
+ def serialize[T: Encoder](child: LogicalPlan): SerializeFromObject = {
SerializeFromObject(encoderFor[T].namedExpressions, child)
}
- def generateObjAttr[T : Encoder]: Attribute = {
+ def generateObjAttr[T: Encoder]: Attribute = {
val enc = encoderFor[T]
val dataType = enc.deserializer.dataType
val nullable = !enc.clsTag.runtimeClass.isPrimitive
@@ -94,7 +94,7 @@ case class SerializeFromObject(
}
object MapPartitions {
- def apply[T : Encoder, U : Encoder](
+ def apply[T: Encoder, U: Encoder](
func: Iterator[T] => Iterator[U],
child: LogicalPlan): LogicalPlan = {
val deserialized = CatalystSerde.deserialize[T](child)
@@ -154,7 +154,7 @@ case class MapPartitionsInR(
}
object MapElements {
- def apply[T : Encoder, U : Encoder](
+ def apply[T: Encoder, U: Encoder](
func: AnyRef,
child: LogicalPlan): LogicalPlan = {
val deserialized = CatalystSerde.deserialize[T](child)
@@ -179,7 +179,7 @@ case class MapElements(
child: LogicalPlan) extends ObjectConsumer with ObjectProducer
object TypedFilter {
- def apply[T : Encoder](func: AnyRef, child: LogicalPlan): TypedFilter = {
+ def apply[T: Encoder](func: AnyRef, child: LogicalPlan): TypedFilter = {
TypedFilter(
func,
implicitly[Encoder[T]].clsTag.runtimeClass,
@@ -259,7 +259,7 @@ object FunctionUtils {
/** Factory for constructing new `AppendColumn` nodes. */
object AppendColumns {
- def apply[T : Encoder, U : Encoder](
+ def apply[T: Encoder, U: Encoder](
func: T => U,
child: LogicalPlan): AppendColumns = {
new AppendColumns(
@@ -271,7 +271,7 @@ object AppendColumns {
child)
}
- def apply[T : Encoder, U : Encoder](
+ def apply[T: Encoder, U: Encoder](
func: T => U,
inputAttributes: Seq[Attribute],
child: LogicalPlan): AppendColumns = {
@@ -319,7 +319,7 @@ case class AppendColumnsWithObject(
/** Factory for constructing new `MapGroups` nodes. */
object MapGroups {
- def apply[K : Encoder, T : Encoder, U : Encoder](
+ def apply[K: Encoder, T: Encoder, U: Encoder](
func: (K, Iterator[T]) => TraversableOnce[U],
groupingAttributes: Seq[Attribute],
dataAttributes: Seq[Attribute],
@@ -475,7 +475,7 @@ case class FlatMapGroupsInR(
/** Factory for constructing new `CoGroup` nodes. */
object CoGroup {
- def apply[K : Encoder, L : Encoder, R : Encoder, OUT : Encoder](
+ def apply[K: Encoder, L: Encoder, R: Encoder, OUT: Encoder](
func: (K, Iterator[L], Iterator[R]) => TraversableOnce[OUT],
leftGroup: Seq[Attribute],
rightGroup: Seq[Attribute],
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index 7375a0bcbae75..56b0971b9faa6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -402,7 +402,9 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
false
} else {
val argsArray: Array[Class[_]] = allArgs.map(_.getClass)
- ClassUtils.isAssignable(argsArray, ctor.getParameterTypes, true /* autoboxing */)
+ // scalastyle:off
+ ClassUtils.isAssignable(argsArray, ctor.getParameterTypes, true /* autoboxing */)
+ // scalastyle:on
}
}.getOrElse(ctors.maxBy(_.getParameterTypes.length)) // fall back to older heuristic
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 746c3e8950f7b..3581db0e3d744 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -580,7 +580,7 @@ object DateTimeUtils {
* The second year of a 400 year period (year 1) starts on day 365.
*/
private[this] def yearBoundary(year: Int): Int = {
- year * 365 + ((year / 4 ) - (year / 100) + (year / 400))
+ year * 365 + ((year / 4) - (year / 100) + (year / 400))
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
index 9c3f6b7c5d245..a8cd2776134b7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
@@ -118,7 +118,7 @@ object NumberConverter {
* unsigned, otherwise it is signed.
* NB: This logic is borrowed from org.apache.hadoop.hive.ql.ud.UDFConv
*/
- def convert(n: Array[Byte], fromBase: Int, toBase: Int ): UTF8String = {
+ def convert(n: Array[Byte], fromBase: Int, toBase: Int): UTF8String = {
if (fromBase < Character.MIN_RADIX || fromBase > Character.MAX_RADIX
|| Math.abs(toBase) < Character.MIN_RADIX
|| Math.abs(toBase) > Character.MAX_RADIX) {
@@ -157,7 +157,7 @@ object NumberConverter {
// Find the first non-zero digit or the last digits if all are zero.
val firstNonZeroPos = {
- val firstNonZero = temp.indexWhere( _ != 0)
+ val firstNonZero = temp.indexWhere(_ != 0)
if (firstNonZero != -1) firstNonZero else temp.length - 1
}
byte2char(Math.abs(toBase), firstNonZeroPos, temp)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
index 4005087dad05a..cffd28b9de018 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
@@ -51,7 +51,7 @@ package object util {
val outStream = new ByteArrayOutputStream
try {
var reading = true
- while ( reading ) {
+ while (reading) {
inStream.read() match {
case -1 => reading = false
case c => outStream.write(c)
@@ -72,7 +72,7 @@ package object util {
val outStream = new ByteArrayOutputStream
try {
var reading = true
- while ( reading ) {
+ while (reading) {
inStream.read() match {
case -1 => reading = false
case c => outStream.write(c)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
index 38c40482fa4d9..d3fe9ee5bc444 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
@@ -100,7 +100,7 @@ case class ArrayType(elementType: DataType, containsNull: Boolean) extends DataT
private[sql] lazy val interpretedOrdering: Ordering[ArrayData] = new Ordering[ArrayData] {
private[this] val elementOrdering: Ordering[Any] = elementType match {
case dt: AtomicType => dt.ordering.asInstanceOf[Ordering[Any]]
- case a : ArrayType => a.interpretedOrdering.asInstanceOf[Ordering[Any]]
+ case a: ArrayType => a.interpretedOrdering.asInstanceOf[Ordering[Any]]
case s: StructType => s.interpretedOrdering.asInstanceOf[Ordering[Any]]
case other =>
throw new IllegalArgumentException(s"Type $other does not support ordered operations")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 1f1fb51addfd8..be8043bda33d5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -380,8 +380,9 @@ final class Decimal extends Ordered[Decimal] with Serializable {
else Decimal(toJavaBigDecimal.remainder(that.toJavaBigDecimal, MATH_CONTEXT))
def remainder(that: Decimal): Decimal = this % that
-
+ // scalastyle:off
def unary_- : Decimal = {
+ // scalastyle:on
if (decimalVal.ne(null)) {
Decimal(-decimalVal, precision, scale)
} else {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala
index 8c766ef829923..a1c4a861c610f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala
@@ -98,5 +98,5 @@ class EncoderErrorMessageSuite extends SparkFunSuite {
s"""array element class: "${clsName[NonEncodable]}""""))
}
- private def clsName[T : ClassTag]: String = implicitly[ClassTag[T]].runtimeClass.getName
+ private def clsName[T: ClassTag]: String = implicitly[ClassTag[T]].runtimeClass.getName
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index 080f11b769388..b63b382d841cc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -114,7 +114,7 @@ object ReferenceValueClass {
class ExpressionEncoderSuite extends PlanTest with AnalysisTest {
OuterScopes.addOuterScope(this)
- implicit def encoder[T : TypeTag]: ExpressionEncoder[T] = ExpressionEncoder()
+ implicit def encoder[T: TypeTag]: ExpressionEncoder[T] = ExpressionEncoder()
// test flat encoders
encodeDecodeTest(false, "primitive boolean")
@@ -189,7 +189,7 @@ class ExpressionEncoderSuite extends PlanTest with AnalysisTest {
encoderFor(Encoders.javaSerialization[JavaSerializable]))
// test product encoders
- private def productTest[T <: Product : ExpressionEncoder](input: T): Unit = {
+ private def productTest[T <: Product: ExpressionEncoder](input: T): Unit = {
encodeDecodeTest(input, input.getClass.getSimpleName)
}
@@ -361,7 +361,7 @@ class ExpressionEncoderSuite extends PlanTest with AnalysisTest {
assert(e.getMessage.contains("Cannot use null as map key"))
}
- private def encodeDecodeTest[T : ExpressionEncoder](
+ private def encodeDecodeTest[T: ExpressionEncoder](
input: T,
testName: String): Unit = {
test(s"encode/decode for $testName: $input") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 4f08031153ab0..918edd1b662d8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -251,7 +251,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
SubstringIndex(Literal("大千世界大千世界"), Literal( "千"), Literal(2)), "大千世界大")
// scalastyle:on
checkEvaluation(
- SubstringIndex(Literal("www||apache||org"), Literal( "||"), Literal(2)), "www||apache")
+ SubstringIndex(Literal("www||apache||org"), Literal("||"), Literal(2)), "www||apache")
}
test("ascii for string") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
index 2420ba513f287..7859efddc468e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
@@ -75,13 +75,13 @@ class PercentileSuite extends SparkFunSuite {
// Run test with Flatten data
val flattenRows = (1 to countForFrequencyTest).flatMap(current =>
- (1 to current).map(y => current )).map(Seq(_))
+ (1 to current).map(y => current)).map(Seq(_))
runTest(agg, flattenRows, expectedPercentilesWithFrquency)
}
private def runTest(agg: Percentile,
- rows : Seq[Seq[Any]],
- expectedPercentiles : Seq[Double]) {
+ rows: Seq[Seq[Any]],
+ expectedPercentiles: Seq[Double]) {
assert(agg.nullable)
val group1 = (0 until rows.length / 2)
val group1Buffer = agg.createAggregationBuffer()
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
index c6345b60b744b..f635807bbef57 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
@@ -103,13 +103,13 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
}
test("e && (!e || f)") {
- checkCondition('e && (!'e || 'f ), 'e && 'f)
+ checkCondition('e && (!'e || 'f), 'e && 'f)
- checkCondition('e && ('f || !'e ), 'e && 'f)
+ checkCondition('e && ('f || !'e), 'e && 'f)
- checkCondition((!'e || 'f ) && 'e, 'f && 'e)
+ checkCondition((!'e || 'f) && 'e, 'f && 'e)
- checkCondition(('f || !'e ) && 'e, 'f && 'e)
+ checkCondition(('f || !'e) && 'e, 'f && 'e)
}
test("a < 1 && (!(a < 1) || f)") {
@@ -127,10 +127,10 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
}
test("a < 1 && ((a >= 1) || f)") {
- checkCondition('a < 1 && ('a >= 1 || 'f ), ('a < 1) && 'f)
+ checkCondition('a < 1 && ('a >= 1 || 'f), ('a < 1) && 'f)
checkCondition('a < 1 && ('f || 'a >= 1), ('a < 1) && 'f)
- checkCondition('a <= 1 && ('a > 1 || 'f ), ('a <= 1) && 'f)
+ checkCondition('a <= 1 && ('a > 1 || 'f), ('a <= 1) && 'f)
checkCondition('a <= 1 && ('f || 'a > 1), ('a <= 1) && 'f)
checkCondition('a > 1 && (('a <= 1) || 'f), ('a > 1) && 'f)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
index 08e58d47e0e25..008571f5b7e4f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
@@ -334,7 +334,7 @@ class ColumnPruningSuite extends PlanTest {
comparePlans(optimized, expected)
}
- implicit private def productEncoder[T <: Product : TypeTag] = ExpressionEncoder[T]()
+ implicit private def productEncoder[T <: Product: TypeTag] = ExpressionEncoder[T]()
private val func = identity[Iterator[OtherTuple]] _
test("Column pruning on MapPartitions") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
index 25c592b9c1dde..e6c05f996d3c7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
@@ -68,7 +68,7 @@ class ConstantFoldingSuite extends PlanTest {
.where(
Literal(1) === Literal(1) &&
Literal(2) > Literal(3) ||
- Literal(3) > Literal(2) )
+ Literal(3) > Literal(2))
.groupBy(
Literal(2) * Literal(3) - Literal(6) / (Literal(4) - Literal(2))
)(Literal(9) / Literal(3) as Symbol("9/3"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSerializationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSerializationSuite.scala
index 3c033ddc374cf..41453d4a0b7fd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSerializationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSerializationSuite.scala
@@ -35,7 +35,7 @@ class EliminateSerializationSuite extends PlanTest {
EliminateSerialization) :: Nil
}
- implicit private def productEncoder[T <: Product : TypeTag] = ExpressionEncoder[T]()
+ implicit private def productEncoder[T <: Product: TypeTag] = ExpressionEncoder[T]()
implicit private def intEncoder = ExpressionEncoder[Int]()
test("back to back serialization") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
index a23d6266b2840..c445443b39325 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
@@ -418,7 +418,7 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
assertEqualPlans(query, expected)
}
- private def assertEqualPlans( plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
+ private def assertEqualPlans(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
val optimized = Optimize.execute(plan1.analyze)
val expected = plan2.analyze
compareJoinOrder(optimized, expected)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
index 605c01b7220d1..b43bc6f6c307a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
@@ -571,7 +571,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
assertEqualPlans(query, expected)
}
- private def assertEqualPlans( plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
+ private def assertEqualPlans(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
val optimized = Optimize.execute(plan1.analyze)
val expected = plan2.analyze
compareJoinOrder(optimized, expected)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/TypedFilterOptimizationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/TypedFilterOptimizationSuite.scala
index 56f096f3ecf8c..af1ca0641e466 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/TypedFilterOptimizationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/TypedFilterOptimizationSuite.scala
@@ -36,7 +36,7 @@ class TypedFilterOptimizationSuite extends PlanTest {
CombineTypedFilters) :: Nil
}
- implicit private def productEncoder[T <: Product : TypeTag] = ExpressionEncoder[T]()
+ implicit private def productEncoder[T <: Product: TypeTag] = ExpressionEncoder[T]()
test("filter after serialize with the same object type") {
val input = LocalRelation('_1.int, '_2.int)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
index 0a18858350e1f..37a753b24637a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
@@ -55,7 +55,7 @@ class ComplexTypesSuite extends PlanTest{
val query = relation
.select(
GetStructField(
- CreateNamedStruct(Seq("att", 'id )),
+ CreateNamedStruct(Seq("att", 'id)),
0,
None) as "outerAtt").analyze
val expected = relation.select('id as "outerAtt").analyze
@@ -301,7 +301,7 @@ class ComplexTypesSuite extends PlanTest{
Literal(2L), 'id,
('id + 3L), ('id + 4L),
('id + 4L), ('id + 5L))),
- 2L ) as "a")
+ 2L) as "a")
.analyze
val expected = relation
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 4d08f016a4a16..196a9f2d6707b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -249,8 +249,8 @@ class ExpressionParserSuite extends PlanTest {
assertEqual("foo(*) over (partition by a, b)", windowed(Seq('a, 'b)))
assertEqual("foo(*) over (distribute by a, b)", windowed(Seq('a, 'b)))
assertEqual("foo(*) over (cluster by a, b)", windowed(Seq('a, 'b)))
- assertEqual("foo(*) over (order by a desc, b asc)", windowed(Seq.empty, Seq('a.desc, 'b.asc )))
- assertEqual("foo(*) over (sort by a desc, b asc)", windowed(Seq.empty, Seq('a.desc, 'b.asc )))
+ assertEqual("foo(*) over (order by a desc, b asc)", windowed(Seq.empty, Seq('a.desc, 'b.asc)))
+ assertEqual("foo(*) over (sort by a desc, b asc)", windowed(Seq.empty, Seq('a.desc, 'b.asc)))
assertEqual("foo(*) over (partition by a, b order by c)", windowed(Seq('a, 'b), Seq('c.asc)))
assertEqual("foo(*) over (distribute by a, b sort by c)", windowed(Seq('a, 'b), Seq('c.asc)))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 7e1f1d83cb3de..7b62777e44154 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -193,7 +193,7 @@ class Column(val expr: Expression) extends Logging {
* results into the correct JVM types.
* @since 1.6.0
*/
- def as[U : Encoder]: TypedColumn[Any, U] = new TypedColumn[Any, U](expr, encoderFor[U])
+ def as[U: Encoder]: TypedColumn[Any, U] = new TypedColumn[Any, U](expr, encoderFor[U])
/**
* Extracts a value or values from a complex type.
@@ -211,7 +211,7 @@ class Column(val expr: Expression) extends Logging {
def apply(extraction: Any): Column = withExpr {
UnresolvedExtractValue(expr, lit(extraction).expr)
}
-
+ // scalastyle:off
/**
* Unary minus, i.e. negate the expression.
* {{{
@@ -242,7 +242,9 @@ class Column(val expr: Expression) extends Logging {
* @group expr_ops
* @since 1.3.0
*/
+
def unary_! : Column = withExpr { Not(expr) }
+ // scalastyle:on
/**
* Equality test.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index 871fff71e5538..0280047774c9c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -391,7 +391,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
df.col(f.name)
}
}
- df.select(projections : _*)
+ df.select(projections: _*)
}
private def fillMap(values: Seq[(String, Any)]): DataFrame = {
@@ -422,7 +422,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
}
}.getOrElse(df.col(f.name))
}
- df.select(projections : _*)
+ df.select(projections: _*)
}
/**
@@ -495,6 +495,6 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
df.col(f.name)
}
}
- df.select(projections : _*)
+ df.select(projections: _*)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index a1d8b7f4af1b5..fa71612379211 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -273,7 +273,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
val params = extraOptions.toMap ++ connectionProperties.asScala.toMap
val options = new JDBCOptions(url, table, params)
val parts: Array[Partition] = predicates.zipWithIndex.map { case (part, i) =>
- JDBCPartition(part, i) : Partition
+ JDBCPartition(part, i): Partition
}
val relation = JDBCRelation(parts, options)(sparkSession)
sparkSession.baseRelationToDataFrame(relation)
@@ -342,7 +342,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
* @since 2.0.0
*/
@scala.annotation.varargs
- def json(paths: String*): DataFrame = format("json").load(paths : _*)
+ def json(paths: String*): DataFrame = format("json").load(paths: _*)
/**
* Loads a `JavaRDD[String]` storing JSON objects (JSON
@@ -542,7 +542,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
* @since 2.0.0
*/
@scala.annotation.varargs
- def csv(paths: String*): DataFrame = format("csv").load(paths : _*)
+ def csv(paths: String*): DataFrame = format("csv").load(paths: _*)
/**
* Loads a Parquet file, returning the result as a `DataFrame`. See the documentation
@@ -632,7 +632,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
* @since 1.6.0
*/
@scala.annotation.varargs
- def text(paths: String*): DataFrame = format("text").load(paths : _*)
+ def text(paths: String*): DataFrame = format("text").load(paths: _*)
/**
* Loads text files and returns a [[Dataset]] of String. See the documentation on the
@@ -666,7 +666,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
@scala.annotation.varargs
def textFile(paths: String*): Dataset[String] = {
assertNoSpecifiedSchema("textFile")
- text(paths : _*).select("value").as[String](sparkSession.implicits.newStringEncoder)
+ text(paths: _*).select("value").as[String](sparkSession.implicits.newStringEncoder)
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 255c4064eb574..59db4e10c9311 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -308,7 +308,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
private def assertNotPartitioned(operation: String): Unit = {
if (partitioningColumns.isDefined) {
- throw new AnalysisException( s"'$operation' does not support partitioning")
+ throw new AnalysisException(s"'$operation' does not support partitioning")
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 7be4aa1ca9562..fde2a1aec4e59 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -413,7 +413,7 @@ class Dataset[T] private[sql](
*/
@Experimental
@InterfaceStability.Evolving
- def as[U : Encoder]: Dataset[U] = Dataset[U](sparkSession, logicalPlan)
+ def as[U: Encoder]: Dataset[U] = Dataset[U](sparkSession, logicalPlan)
/**
* Converts this strongly typed collection of data to generic `DataFrame` with columns renamed.
@@ -438,7 +438,7 @@ class Dataset[T] private[sql](
val newCols = logicalPlan.output.zip(colNames).map { case (oldAttribute, newName) =>
Column(oldAttribute).as(newName)
}
- select(newCols : _*)
+ select(newCols: _*)
}
/**
@@ -1083,7 +1083,7 @@ class Dataset[T] private[sql](
*/
@scala.annotation.varargs
def sortWithinPartitions(sortCol: String, sortCols: String*): Dataset[T] = {
- sortWithinPartitions((sortCol +: sortCols).map(Column(_)) : _*)
+ sortWithinPartitions((sortCol +: sortCols).map(Column(_)): _*)
}
/**
@@ -1113,7 +1113,7 @@ class Dataset[T] private[sql](
*/
@scala.annotation.varargs
def sort(sortCol: String, sortCols: String*): Dataset[T] = {
- sort((sortCol +: sortCols).map(apply) : _*)
+ sort((sortCol +: sortCols).map(apply): _*)
}
/**
@@ -1138,7 +1138,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@scala.annotation.varargs
- def orderBy(sortCol: String, sortCols: String*): Dataset[T] = sort(sortCol, sortCols : _*)
+ def orderBy(sortCol: String, sortCols: String*): Dataset[T] = sort(sortCol, sortCols: _*)
/**
* Returns a new Dataset sorted by the given expressions.
@@ -1148,7 +1148,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@scala.annotation.varargs
- def orderBy(sortExprs: Column*): Dataset[T] = sort(sortExprs : _*)
+ def orderBy(sortExprs: Column*): Dataset[T] = sort(sortExprs: _*)
/**
* Selects column based on the column name and return it as a [[Column]].
@@ -1254,7 +1254,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@scala.annotation.varargs
- def select(col: String, cols: String*): DataFrame = select((col +: cols).map(Column(_)) : _*)
+ def select(col: String, cols: String*): DataFrame = select((col +: cols).map(Column(_)): _*)
/**
* Selects a set of SQL expressions. This is a variant of `select` that accepts
@@ -1659,7 +1659,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
def agg(aggExpr: (String, String), aggExprs: (String, String)*): DataFrame = {
- groupBy().agg(aggExpr, aggExprs : _*)
+ groupBy().agg(aggExpr, aggExprs: _*)
}
/**
@@ -1700,7 +1700,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@scala.annotation.varargs
- def agg(expr: Column, exprs: Column*): DataFrame = groupBy().agg(expr, exprs : _*)
+ def agg(expr: Column, exprs: Column*): DataFrame = groupBy().agg(expr, exprs: _*)
/**
* Returns a new Dataset by taking the first `n` rows. The difference between this function
@@ -1859,7 +1859,7 @@ class Dataset[T] private[sql](
*/
def randomSplitAsList(weights: Array[Double], seed: Long): java.util.List[Dataset[T]] = {
val values = randomSplit(weights, seed)
- java.util.Arrays.asList(values : _*)
+ java.util.Arrays.asList(values: _*)
}
/**
@@ -1911,7 +1911,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0")
- def explode[A <: Product : TypeTag](input: Column*)(f: Row => TraversableOnce[A]): DataFrame = {
+ def explode[A <: Product: TypeTag](input: Column*)(f: Row => TraversableOnce[A]): DataFrame = {
val elementSchema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
val convert = CatalystTypeConverters.createToCatalystConverter(elementSchema)
@@ -1948,7 +1948,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
@deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0")
- def explode[A, B : TypeTag](inputColumn: String, outputColumn: String)(f: A => TraversableOnce[B])
+ def explode[A, B: TypeTag](inputColumn: String, outputColumn: String)(f: A => TraversableOnce[B])
: DataFrame = {
val dataType = ScalaReflection.schemaFor[B].dataType
val attributes = AttributeReference(outputColumn, dataType)() :: Nil
@@ -1986,7 +1986,7 @@ class Dataset[T] private[sql](
Column(field)
}
}
- select(columns : _*)
+ select(columns: _*)
} else {
select(Column("*"), col.as(colName))
}
@@ -2018,7 +2018,7 @@ class Dataset[T] private[sql](
Column(col)
}
}
- select(columns : _*)
+ select(columns: _*)
} else {
toDF()
}
@@ -2035,7 +2035,7 @@ class Dataset[T] private[sql](
* @since 2.0.0
*/
def drop(colName: String): DataFrame = {
- drop(Seq(colName) : _*)
+ drop(Seq(colName): _*)
}
/**
@@ -2082,7 +2082,7 @@ class Dataset[T] private[sql](
val colsAfterDrop = attrs.filter { attr =>
attr != expression
}.map(attr => Column(attr))
- select(colsAfterDrop : _*)
+ select(colsAfterDrop: _*)
}
/**
@@ -2302,7 +2302,7 @@ class Dataset[T] private[sql](
*/
@Experimental
@InterfaceStability.Evolving
- def map[U : Encoder](func: T => U): Dataset[U] = withTypedPlan {
+ def map[U: Encoder](func: T => U): Dataset[U] = withTypedPlan {
MapElements[T, U](func, logicalPlan)
}
@@ -2331,7 +2331,7 @@ class Dataset[T] private[sql](
*/
@Experimental
@InterfaceStability.Evolving
- def mapPartitions[U : Encoder](func: Iterator[T] => Iterator[U]): Dataset[U] = {
+ def mapPartitions[U: Encoder](func: Iterator[T] => Iterator[U]): Dataset[U] = {
new Dataset[U](
sparkSession,
MapPartitions[T, U](func, logicalPlan),
@@ -2379,7 +2379,7 @@ class Dataset[T] private[sql](
*/
@Experimental
@InterfaceStability.Evolving
- def flatMap[U : Encoder](func: T => TraversableOnce[U]): Dataset[U] =
+ def flatMap[U: Encoder](func: T => TraversableOnce[U]): Dataset[U] =
mapPartitions(_.flatMap(func))
/**
@@ -2457,7 +2457,7 @@ class Dataset[T] private[sql](
* @group action
* @since 1.6.0
*/
- def takeAsList(n: Int): java.util.List[T] = java.util.Arrays.asList(take(n) : _*)
+ def takeAsList(n: Int): java.util.List[T] = java.util.Arrays.asList(take(n): _*)
/**
* Returns an array that contains all rows in this Dataset.
@@ -2483,7 +2483,7 @@ class Dataset[T] private[sql](
*/
def collectAsList(): java.util.List[T] = withAction("collectAsList", queryExecution) { plan =>
val values = collectFromPlan(plan)
- java.util.Arrays.asList(values : _*)
+ java.util.Arrays.asList(values: _*)
}
/**
@@ -2954,12 +2954,12 @@ class Dataset[T] private[sql](
}
/** A convenient function to wrap a logical plan and produce a Dataset. */
- @inline private def withTypedPlan[U : Encoder](logicalPlan: LogicalPlan): Dataset[U] = {
+ @inline private def withTypedPlan[U: Encoder](logicalPlan: LogicalPlan): Dataset[U] = {
Dataset(sparkSession, logicalPlan)
}
/** A convenient function to wrap a set based logical plan and produce a Dataset. */
- @inline private def withSetOperator[U : Encoder](logicalPlan: LogicalPlan): Dataset[U] = {
+ @inline private def withSetOperator[U: Encoder](logicalPlan: LogicalPlan): Dataset[U] = {
if (classTag.runtimeClass.isAssignableFrom(classOf[Row])) {
// Set operators widen types (change the schema), so we cannot reuse the row encoder.
Dataset.ofRows(sparkSession, logicalPlan).asInstanceOf[Dataset[U]]
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala b/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala
index 582d4a3670b8e..2d1f4dd658fb7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DatasetHolder.scala
@@ -41,5 +41,5 @@ case class DatasetHolder[T] private[sql](private val ds: Dataset[T]) {
// `rdd.toDF("1")` as invoking this toDF and then apply on the returned DataFrame.
def toDF(): DataFrame = ds.toDF()
- def toDF(colNames: String*): DataFrame = ds.toDF(colNames : _*)
+ def toDF(colNames: String*): DataFrame = ds.toDF(colNames: _*)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
index cb42e9e4560cf..7bf258c39f7e0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
@@ -60,7 +60,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
*
* @since 1.6.0
*/
- def keyAs[L : Encoder]: KeyValueGroupedDataset[L, V] =
+ def keyAs[L: Encoder]: KeyValueGroupedDataset[L, V] =
new KeyValueGroupedDataset(
encoderFor[L],
vExprEnc,
@@ -79,7 +79,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
*
* @since 2.1.0
*/
- def mapValues[W : Encoder](func: V => W): KeyValueGroupedDataset[K, W] = {
+ def mapValues[W: Encoder](func: V => W): KeyValueGroupedDataset[K, W] = {
val withNewData = AppendColumns(func, dataAttributes, logicalPlan)
val projected = Project(withNewData.newColumns ++ groupingAttributes, withNewData)
val executed = sparkSession.sessionState.executePlan(projected)
@@ -142,7 +142,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
*
* @since 1.6.0
*/
- def flatMapGroups[U : Encoder](f: (K, Iterator[V]) => TraversableOnce[U]): Dataset[U] = {
+ def flatMapGroups[U: Encoder](f: (K, Iterator[V]) => TraversableOnce[U]): Dataset[U] = {
Dataset[U](
sparkSession,
MapGroups(
@@ -193,7 +193,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
*
* @since 1.6.0
*/
- def mapGroups[U : Encoder](f: (K, Iterator[V]) => U): Dataset[U] = {
+ def mapGroups[U: Encoder](f: (K, Iterator[V]) => U): Dataset[U] = {
val func = (key: K, it: Iterator[V]) => Iterator(f(key, it))
flatMapGroups(func)
}
@@ -533,7 +533,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
*
* @since 1.6.0
*/
- def cogroup[U, R : Encoder](
+ def cogroup[U, R: Encoder](
other: KeyValueGroupedDataset[K, U])(
f: (K, Iterator[V], Iterator[U]) => TraversableOnce[R]): Dataset[R] = {
implicit val uEncoder = other.vExprEnc
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
index 147b549964913..d9878a825c133 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
@@ -246,7 +246,7 @@ class RelationalGroupedDataset protected[sql](
*/
@scala.annotation.varargs
def mean(colNames: String*): DataFrame = {
- aggregateNumericColumns(colNames : _*)(Average)
+ aggregateNumericColumns(colNames: _*)(Average)
}
/**
@@ -258,7 +258,7 @@ class RelationalGroupedDataset protected[sql](
*/
@scala.annotation.varargs
def max(colNames: String*): DataFrame = {
- aggregateNumericColumns(colNames : _*)(Max)
+ aggregateNumericColumns(colNames: _*)(Max)
}
/**
@@ -270,7 +270,7 @@ class RelationalGroupedDataset protected[sql](
*/
@scala.annotation.varargs
def avg(colNames: String*): DataFrame = {
- aggregateNumericColumns(colNames : _*)(Average)
+ aggregateNumericColumns(colNames: _*)(Average)
}
/**
@@ -282,7 +282,7 @@ class RelationalGroupedDataset protected[sql](
*/
@scala.annotation.varargs
def min(colNames: String*): DataFrame = {
- aggregateNumericColumns(colNames : _*)(Min)
+ aggregateNumericColumns(colNames: _*)(Min)
}
/**
@@ -294,7 +294,7 @@ class RelationalGroupedDataset protected[sql](
*/
@scala.annotation.varargs
def sum(colNames: String*): DataFrame = {
- aggregateNumericColumns(colNames : _*)(Sum)
+ aggregateNumericColumns(colNames: _*)(Sum)
}
/**
@@ -419,7 +419,7 @@ class RelationalGroupedDataset protected[sql](
outputSchema: StructType): DataFrame = {
val groupingNamedExpressions = groupingExprs.map(alias)
val groupingCols = groupingNamedExpressions.map(Column(_))
- val groupingDataFrame = df.select(groupingCols : _*)
+ val groupingDataFrame = df.select(groupingCols: _*)
val groupingAttributes = groupingNamedExpressions.map(_.toAttribute)
Dataset.ofRows(
df.sparkSession,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 7fde6e9469e5e..2c6b434ff3920 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -268,7 +268,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
*/
@Experimental
@InterfaceStability.Evolving
- def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = {
+ def createDataFrame[A <: Product: TypeTag](rdd: RDD[A]): DataFrame = {
sparkSession.createDataFrame(rdd)
}
@@ -281,7 +281,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
*/
@Experimental
@InterfaceStability.Evolving
- def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = {
+ def createDataFrame[A <: Product: TypeTag](data: Seq[A]): DataFrame = {
sparkSession.createDataFrame(data)
}
@@ -373,7 +373,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
*/
@Experimental
@InterfaceStability.Evolving
- def createDataset[T : Encoder](data: Seq[T]): Dataset[T] = {
+ def createDataset[T: Encoder](data: Seq[T]): Dataset[T] = {
sparkSession.createDataset(data)
}
@@ -388,7 +388,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @group dataset
*/
@Experimental
- def createDataset[T : Encoder](data: RDD[T]): Dataset[T] = {
+ def createDataset[T: Encoder](data: RDD[T]): Dataset[T] = {
sparkSession.createDataset(data)
}
@@ -411,7 +411,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
*/
@Experimental
@InterfaceStability.Evolving
- def createDataset[T : Encoder](data: java.util.List[T]): Dataset[T] = {
+ def createDataset[T: Encoder](data: java.util.List[T]): Dataset[T] = {
sparkSession.createDataset(data)
}
@@ -803,7 +803,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
if (paths.isEmpty) {
emptyDataFrame
} else {
- read.parquet(paths : _*)
+ read.parquet(paths: _*)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
index 86574e2f71d92..c95eb1be98e96 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
@@ -162,14 +162,14 @@ abstract class SQLImplicits extends LowPrioritySQLImplicits {
* @since 1.6.1
* @deprecated use [[newSequenceEncoder]]
*/
- def newProductSeqEncoder[A <: Product : TypeTag]: Encoder[Seq[A]] = ExpressionEncoder()
+ def newProductSeqEncoder[A <: Product: TypeTag]: Encoder[Seq[A]] = ExpressionEncoder()
/** @since 2.2.0 */
- implicit def newSequenceEncoder[T <: Seq[_] : TypeTag]: Encoder[T] = ExpressionEncoder()
+ implicit def newSequenceEncoder[T <: Seq[_]: TypeTag]: Encoder[T] = ExpressionEncoder()
// Maps
/** @since 2.3.0 */
- implicit def newMapEncoder[T <: Map[_, _] : TypeTag]: Encoder[T] = ExpressionEncoder()
+ implicit def newMapEncoder[T <: Map[_, _]: TypeTag]: Encoder[T] = ExpressionEncoder()
// Arrays
@@ -198,7 +198,7 @@ abstract class SQLImplicits extends LowPrioritySQLImplicits {
implicit def newStringArrayEncoder: Encoder[Array[String]] = ExpressionEncoder()
/** @since 1.6.1 */
- implicit def newProductArrayEncoder[A <: Product : TypeTag]: Encoder[Array[A]] =
+ implicit def newProductArrayEncoder[A <: Product: TypeTag]: Encoder[Array[A]] =
ExpressionEncoder()
/**
@@ -206,7 +206,7 @@ abstract class SQLImplicits extends LowPrioritySQLImplicits {
*
* @since 1.6.0
*/
- implicit def rddToDatasetHolder[T : Encoder](rdd: RDD[T]): DatasetHolder[T] = {
+ implicit def rddToDatasetHolder[T: Encoder](rdd: RDD[T]): DatasetHolder[T] = {
DatasetHolder(_sqlContext.createDataset(rdd))
}
@@ -214,7 +214,7 @@ abstract class SQLImplicits extends LowPrioritySQLImplicits {
* Creates a [[Dataset]] from a local Seq.
* @since 1.6.0
*/
- implicit def localSeqToDatasetHolder[T : Encoder](s: Seq[T]): DatasetHolder[T] = {
+ implicit def localSeqToDatasetHolder[T: Encoder](s: Seq[T]): DatasetHolder[T] = {
DatasetHolder(_sqlContext.createDataset(s))
}
@@ -235,6 +235,6 @@ abstract class SQLImplicits extends LowPrioritySQLImplicits {
*/
trait LowPrioritySQLImplicits {
/** @since 1.6.0 */
- implicit def newProductEncoder[T <: Product : TypeTag]: Encoder[T] = Encoders.product[T]
+ implicit def newProductEncoder[T <: Product: TypeTag]: Encoder[T] = Encoders.product[T]
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 2c38f7d7c88da..3a28db867c30a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -282,7 +282,7 @@ class SparkSession private(
*/
@Experimental
@InterfaceStability.Evolving
- def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = {
+ def createDataFrame[A <: Product: TypeTag](rdd: RDD[A]): DataFrame = {
SparkSession.setActiveSession(this)
val encoder = Encoders.product[A]
Dataset.ofRows(self, ExternalRDD(rdd, self)(encoder))
@@ -296,7 +296,7 @@ class SparkSession private(
*/
@Experimental
@InterfaceStability.Evolving
- def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = {
+ def createDataFrame[A <: Product: TypeTag](data: Seq[A]): DataFrame = {
SparkSession.setActiveSession(this)
val schema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
val attributeSeq = schema.toAttributes
@@ -454,7 +454,7 @@ class SparkSession private(
*/
@Experimental
@InterfaceStability.Evolving
- def createDataset[T : Encoder](data: Seq[T]): Dataset[T] = {
+ def createDataset[T: Encoder](data: Seq[T]): Dataset[T] = {
val enc = encoderFor[T]
val attributes = enc.schema.toAttributes
val encoded = data.map(d => enc.toRow(d).copy())
@@ -473,7 +473,7 @@ class SparkSession private(
*/
@Experimental
@InterfaceStability.Evolving
- def createDataset[T : Encoder](data: RDD[T]): Dataset[T] = {
+ def createDataset[T: Encoder](data: RDD[T]): Dataset[T] = {
Dataset[T](self, ExternalRDD(data, self))
}
@@ -495,7 +495,7 @@ class SparkSession private(
*/
@Experimental
@InterfaceStability.Evolving
- def createDataset[T : Encoder](data: java.util.List[T]): Dataset[T] = {
+ def createDataset[T: Encoder](data: java.util.List[T]): Dataset[T] = {
createDataset(data.asScala)
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index ad01b889429c7..0d2383420ce72 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -90,7 +90,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
* @since 2.2.0
*/
def register(name: String, udf: UserDefinedFunction): UserDefinedFunction = {
- def builder(children: Seq[Expression]) = udf.apply(children.map(Column.apply) : _*).expr
+ def builder(children: Seq[Expression]) = udf.apply(children.map(Column.apply): _*).expr
functionRegistry.createOrReplaceTempFunction(name, builder)
udf
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala
index 717758fdf716f..3b4e64c624d24 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.expressions.Aggregator
import org.apache.spark.sql.types._
object TypedAggregateExpression {
- def apply[BUF : Encoder, OUT : Encoder](
+ def apply[BUF: Encoder, OUT: Encoder](
aggregator: Aggregator[_, BUF, OUT]): TypedAggregateExpression = {
val bufferEncoder = encoderFor[BUF]
val bufferSerializer = bufferEncoder.namedExpressions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
index ab26f2affbce5..f1dee6947bc4c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
@@ -138,7 +138,9 @@ case class InsertIntoHadoopFsRelationCommand(
AlterTableDropPartitionCommand(
catalogTable.get.identifier, deletedPartitions.toSeq,
ifExists = true, purge = false,
+ // scalastyle:off
retainData = true /* already deleted */).run(sparkSession)
+ // scalastyle:on
}
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
index 0b805e4362883..d6197e062341c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
@@ -582,7 +582,7 @@ private[parquet] object ParquetSchemaConverter {
}
}
- private def computeMinBytesForPrecision(precision : Int) : Int = {
+ private def computeMinBytesForPrecision(precision: Int): Int = {
var numBytes = 1
while (math.pow(2.0, 8 * numBytes - 1) < math.pow(10.0, precision)) {
numBytes += 1
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
index f380986951317..b7499e4937c8e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
@@ -32,8 +32,8 @@ import org.apache.spark.util.CompletionIterator
* materialize the right RDD (in case of the right RDD is nondeterministic).
*/
class UnsafeCartesianRDD(
- left : RDD[UnsafeRow],
- right : RDD[UnsafeRow],
+ left: RDD[UnsafeRow],
+ right: RDD[UnsafeRow],
numFieldsOfRight: Int,
spillThreshold: Int)
extends CartesianRDD[UnsafeRow, UnsafeRow](left.sparkContext, left, right) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
index 3c702856114f9..0994165a86fcd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
@@ -189,7 +189,7 @@ private[joins] class UnsafeHashedRelation(
private def write(
writeInt: (Int) => Unit,
writeLong: (Long) => Unit,
- writeBuffer: (Array[Byte], Int, Int) => Unit) : Unit = {
+ writeBuffer: (Array[Byte], Int, Int) => Unit): Unit = {
writeInt(numFields)
// TODO: move these into BytesToBytesMap
writeLong(binaryMap.numKeys())
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
index cdb755edc79a1..b48ad96845078 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
@@ -95,7 +95,7 @@ object FrequentItems extends Logging {
(name, originalSchema.fields(index).dataType)
}.toArray
- val freqItems = df.select(cols.map(Column(_)) : _*).rdd.aggregate(countMaps)(
+ val freqItems = df.select(cols.map(Column(_)): _*).rdd.aggregate(countMaps)(
seqOp = (counts, row) => {
var i = 0
while (i < numCols) {
@@ -116,7 +116,7 @@ object FrequentItems extends Logging {
}
)
val justItems = freqItems.map(m => m.baseMap.keys.toArray)
- val resultRow = Row(justItems : _*)
+ val resultRow = Row(justItems: _*)
// append frequent Items to the column name for easy debugging
val outputCols = colInfo.map { v =>
StructField(v._1 + "_freqItems", ArrayType(v._2, false))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
index 408c8f81f17ba..403e6244ba5ed 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
@@ -39,7 +39,7 @@ import org.apache.spark.sql.SparkSession
* compact log files every 10 batches by default into a big file. When
* doing a compaction, it will read all old log files and merge them with the new batch.
*/
-abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
+abstract class CompactibleFileStreamLog[T <: AnyRef: ClassTag](
metadataLogVersion: Int,
sparkSession: SparkSession,
path: String)
@@ -305,7 +305,7 @@ object CompactibleFileStreamLog {
* Derives a compact interval from the latest compact batch id and
* a default compact interval.
*/
- def deriveCompactInterval(defaultInterval: Int, latestCompactBatchId: Int) : Int = {
+ def deriveCompactInterval(defaultInterval: Int, latestCompactBatchId: Int): Int = {
if (latestCompactBatchId + 1 <= defaultInterval) {
latestCompactBatchId + 1
} else if (defaultInterval < (latestCompactBatchId + 1) / 2) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ForeachSink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ForeachSink.scala
index de09fb568d2a6..a4cbc712a472e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ForeachSink.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ForeachSink.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.encoders.encoderFor
* @param writer The [[ForeachWriter]] to process all data.
* @tparam T The expected type of the sink.
*/
-class ForeachSink[T : Encoder](writer: ForeachWriter[T]) extends Sink with Serializable {
+class ForeachSink[T: Encoder](writer: ForeachWriter[T]) extends Sink with Serializable {
override def addBatch(batchId: Long, data: DataFrame): Unit = {
// This logic should've been as simple as:
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
index 46bfc297931fb..5996de9c13f18 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/HDFSMetadataLog.scala
@@ -45,7 +45,7 @@ import org.apache.spark.sql.SparkSession
* Note: [[HDFSMetadataLog]] doesn't support S3-like file systems as they don't guarantee listing
* files in a directory always shows the latest files.
*/
-class HDFSMetadataLog[T <: AnyRef : ClassTag](sparkSession: SparkSession, path: String)
+class HDFSMetadataLog[T <: AnyRef: ClassTag](sparkSession: SparkSession, path: String)
extends MetadataLog[T] with Logging {
private implicit val formats = Serialization.formats(NoTypeHints)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/LongOffset.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/LongOffset.scala
index 5f0b195fcfcb8..c47b137040fb2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/LongOffset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/LongOffset.scala
@@ -34,7 +34,7 @@ object LongOffset {
* LongOffset factory from serialized offset.
* @return new LongOffset
*/
- def apply(offset: SerializedOffset) : LongOffset = new LongOffset(offset.json.toLong)
+ def apply(offset: SerializedOffset): LongOffset = new LongOffset(offset.json.toLong)
/**
* Convert generic Offset to LongOffset if possible.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Source.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Source.scala
index 311942f6dbd84..9167923f8ca8f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Source.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Source.scala
@@ -60,7 +60,7 @@ trait Source {
* Informs the source that Spark has completed processing all data for offsets less than or
* equal to `end` and will only request offsets greater than `end` in the future.
*/
- def commit(end: Offset) : Unit = {}
+ def commit(end: Offset): Unit = {}
/** Stop this source and free any resources it has allocated. */
def stop(): Unit
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
index 4979873ee3c7f..c61dac7e9d282 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
@@ -38,7 +38,7 @@ object MemoryStream {
protected val currentBlockId = new AtomicInteger(0)
protected val memoryStreamId = new AtomicInteger(0)
- def apply[A : Encoder](implicit sqlContext: SQLContext): MemoryStream[A] =
+ def apply[A: Encoder](implicit sqlContext: SQLContext): MemoryStream[A] =
new MemoryStream[A](memoryStreamId.getAndIncrement(), sqlContext)
}
@@ -47,7 +47,7 @@ object MemoryStream {
* is primarily intended for use in unit tests as it can only replay data when the object is still
* available.
*/
-case class MemoryStream[A : Encoder](id: Int, sqlContext: SQLContext)
+case class MemoryStream[A: Encoder](id: Int, sqlContext: SQLContext)
extends Source with Logging {
protected val encoder = encoderFor[A]
protected val logicalPlan = StreamingExecutionRelation(this)
@@ -68,7 +68,7 @@ case class MemoryStream[A : Encoder](id: Int, sqlContext: SQLContext)
* -1 is used in calculations below and isn't just an arbitrary constant.
*/
@GuardedBy("this")
- protected var lastOffsetCommitted : LongOffset = new LongOffset(-1)
+ protected var lastOffsetCommitted: LongOffset = new LongOffset(-1)
def schema: StructType = encoder.schema
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/socket.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/socket.scala
index 8e63207959575..28d30db349724 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/socket.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/socket.scala
@@ -65,7 +65,7 @@ class TextSocketSource(host: String, port: Int, includeTimestamp: Boolean, sqlCo
protected var currentOffset: LongOffset = new LongOffset(-1)
@GuardedBy("this")
- protected var lastOffsetCommitted : LongOffset = new LongOffset(-1)
+ protected var lastOffsetCommitted: LongOffset = new LongOffset(-1)
initialize()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
index bae7a15165e43..64076b9a7c234 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/HDFSBackedStateStoreProvider.scala
@@ -205,7 +205,7 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
indexOrdinal: Option[Int], // for sorting the data
storeConf: StateStoreConf,
hadoopConf: Configuration): Unit = {
- this.stateStoreId_ = stateStoreId
+ this._stateStoreId = stateStoreId
this.keySchema = keySchema
this.valueSchema = valueSchema
this.storeConf = storeConf
@@ -213,7 +213,7 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
fs.mkdirs(baseDir)
}
- override def stateStoreId: StateStoreId = stateStoreId_
+ override def stateStoreId: StateStoreId = _stateStoreId
/** Do maintenance backing data files, including creating snapshots and cleaning up old files */
override def doMaintenance(): Unit = {
@@ -237,7 +237,7 @@ private[state] class HDFSBackedStateStoreProvider extends StateStoreProvider wit
/* Internal fields and methods */
- @volatile private var stateStoreId_ : StateStoreId = _
+ @volatile private var _stateStoreId: StateStoreId = _
@volatile private var keySchema: StructType = _
@volatile private var valueSchema: StructType = _
@volatile private var storeConf: StateStoreConf = _
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
index cd79128d8f375..6667e9f6ec7f2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Window.scala
@@ -44,7 +44,7 @@ object Window {
*/
@scala.annotation.varargs
def partitionBy(colName: String, colNames: String*): WindowSpec = {
- spec.partitionBy(colName, colNames : _*)
+ spec.partitionBy(colName, colNames: _*)
}
/**
@@ -53,7 +53,7 @@ object Window {
*/
@scala.annotation.varargs
def partitionBy(cols: Column*): WindowSpec = {
- spec.partitionBy(cols : _*)
+ spec.partitionBy(cols: _*)
}
/**
@@ -62,7 +62,7 @@ object Window {
*/
@scala.annotation.varargs
def orderBy(colName: String, colNames: String*): WindowSpec = {
- spec.orderBy(colName, colNames : _*)
+ spec.orderBy(colName, colNames: _*)
}
/**
@@ -71,7 +71,7 @@ object Window {
*/
@scala.annotation.varargs
def orderBy(cols: Column*): WindowSpec = {
- spec.orderBy(cols : _*)
+ spec.orderBy(cols: _*)
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 839cbf42024e3..6ae6ae594af22 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -106,7 +106,7 @@ object functions {
* @group normal_funcs
* @since 2.2.0
*/
- def typedLit[T : TypeTag](literal: T): Column = literal match {
+ def typedLit[T: TypeTag](literal: T): Column = literal match {
case c: Column => c
case s: Symbol => new ColumnName(s.name)
case _ => Column(Literal.create(literal))
@@ -373,7 +373,7 @@ object functions {
*/
@scala.annotation.varargs
def countDistinct(columnName: String, columnNames: String*): Column =
- countDistinct(Column(columnName), columnNames.map(Column.apply) : _*)
+ countDistinct(Column(columnName), columnNames.map(Column.apply): _*)
/**
* Aggregate function: returns the population covariance for two columns.
@@ -509,7 +509,7 @@ object functions {
* @since 2.0.0
*/
def grouping_id(colName: String, colNames: String*): Column = {
- grouping_id((Seq(colName) ++ colNames).map(n => Column(n)) : _*)
+ grouping_id((Seq(colName) ++ colNames).map(n => Column(n)): _*)
}
/**
@@ -992,7 +992,7 @@ object functions {
*/
@scala.annotation.varargs
def array(colName: String, colNames: String*): Column = {
- array((colName +: colNames).map(col) : _*)
+ array((colName +: colNames).map(col): _*)
}
/**
@@ -1228,7 +1228,7 @@ object functions {
*/
@scala.annotation.varargs
def struct(colName: String, colNames: String*): Column = {
- struct((colName +: colNames).map(col) : _*)
+ struct((colName +: colNames).map(col): _*)
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/AggregatedDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/AggregatedDialect.scala
index 467d8d62d1b7f..d2c31d6e04107 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/AggregatedDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/AggregatedDialect.scala
@@ -30,7 +30,7 @@ private class AggregatedDialect(dialects: List[JdbcDialect]) extends JdbcDialect
require(dialects.nonEmpty)
- override def canHandle(url : String): Boolean =
+ override def canHandle(url: String): Boolean =
dialects.map(_.canHandle(url)).reduce(_ && _)
override def getCatalystType(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
index a86a86d408906..622789723b455 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
@@ -32,7 +32,7 @@ import org.apache.spark.sql.types._
*/
@DeveloperApi
@InterfaceStability.Evolving
-case class JdbcType(databaseTypeDefinition : String, jdbcNullType : Int)
+case class JdbcType(databaseTypeDefinition: String, jdbcNullType: Int)
/**
* :: DeveloperApi ::
@@ -62,7 +62,7 @@ abstract class JdbcDialect extends Serializable {
* @return True if the dialect can be applied on the given jdbc url.
* @throws NullPointerException if the url is null.
*/
- def canHandle(url : String): Boolean
+ def canHandle(url: String): Boolean
/**
* Get the custom datatype mapping for the given jdbc meta information.
@@ -153,7 +153,7 @@ object JdbcDialects {
*
* @param dialect The new dialect.
*/
- def registerDialect(dialect: JdbcDialect) : Unit = {
+ def registerDialect(dialect: JdbcDialect): Unit = {
dialects = dialect :: dialects.filterNot(_ == dialect)
}
@@ -162,7 +162,7 @@ object JdbcDialects {
*
* @param dialect The jdbc dialect.
*/
- def unregisterDialect(dialect : JdbcDialect) : Unit = {
+ def unregisterDialect(dialect: JdbcDialect): Unit = {
dialects = dialects.filterNot(_ == dialect)
}
@@ -193,5 +193,5 @@ object JdbcDialects {
* NOOP dialect object, always returning the neutral element.
*/
private object NoopDialect extends JdbcDialect {
- override def canHandle(url : String): Boolean = true
+ override def canHandle(url: String): Boolean = true
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
index b2cff7877d8b5..634f643b9b0fd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.types.{BooleanType, DataType, LongType, MetadataBuil
private case object MySQLDialect extends JdbcDialect {
- override def canHandle(url : String): Boolean = url.startsWith("jdbc:mysql")
+ override def canHandle(url: String): Boolean = url.startsWith("jdbc:mysql")
override def getCatalystType(
sqlType: Int, typeName: String, size: Int, md: MetadataBuilder): Option[DataType] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
index fb590e7df996b..78dfae96cb340 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/progress.scala
@@ -116,7 +116,7 @@ class StreamingQueryProgress private[sql](
def safeMapToJValue[T](map: ju.Map[String, T], valueToJValue: T => JValue): JValue = {
if (map.isEmpty) return JNothing
val keys = map.asScala.keySet.toSeq.sorted
- keys.map { k => k -> valueToJValue(map.get(k)) : JObject }.reduce(_ ~ _)
+ keys.map { k => k -> valueToJValue(map.get(k)): JObject }.reduce(_ ~ _)
}
("id" -> JString(id.toString)) ~
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
index 0e7eaa9e88d57..ddb6af6409c33 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
@@ -102,7 +102,7 @@ object SeqAgg extends Aggregator[AggData, Seq[Int], Seq[(Int, Int)]] {
}
-class ParameterizedTypeSum[IN, OUT : Numeric : Encoder](f: IN => OUT)
+class ParameterizedTypeSum[IN, OUT: Numeric: Encoder](f: IN => OUT)
extends Aggregator[IN, OUT, OUT] {
private val numeric = implicitly[Numeric[OUT]]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 87b7b090de3bf..9894d10d3ac15 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -213,7 +213,7 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
checkDataset(
ds.select(
expr("_1").as[String],
- expr("_2").as[Int]) : Dataset[(String, Int)],
+ expr("_2").as[Int]): Dataset[(String, Int)],
("a", 1), ("b", 2), ("c", 3))
}
@@ -577,7 +577,7 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
}
test("SPARK-14696: implicit encoders for boxed types") {
- assert(spark.range(1).map { i => i : java.lang.Long }.head == 0L)
+ assert(spark.range(1).map { i => i: java.lang.Long }.head == 0L)
}
test("SPARK-11894: Incorrect results are returned when using null") {
@@ -727,9 +727,9 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
}
test("SPARK-14554: Dataset.map may generate wrong java code for wide table") {
- val wideDF = spark.range(10).select(Seq.tabulate(1000) {i => ('id + i).as(s"c$i")} : _*)
+ val wideDF = spark.range(10).select(Seq.tabulate(1000) {i => ('id + i).as(s"c$i")}: _*)
// Make sure the generated code for this plan can compile and execute.
- checkDataset(wideDF.map(_.getLong(0)), 0L until 10 : _*)
+ checkDataset(wideDF.map(_.getLong(0)), 0L until 10: _*)
}
test("SPARK-14838: estimating sizeInBytes in operators with ObjectProducer shouldn't fail") {
@@ -1144,7 +1144,7 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
test ("SPARK-17460: the sizeInBytes in Statistics shouldn't overflow to a negative number") {
// Since the sizeInBytes in Statistics could exceed the limit of an Int, we should use BigInt
// instead of Int for avoiding possible overflow.
- val ds = (0 to 10000).map( i =>
+ val ds = (0 to 10000).map(i =>
(i, Seq((i, Seq((i, "This is really not that long of a string")))))).toDS()
val sizeInBytes = ds.logicalPlan.stats.sizeInBytes
// sizeInBytes is 2404280404, before the fix, it overflows to a negative number
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index f9808834df4a5..cddab52c28112 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -89,7 +89,7 @@ abstract class QueryTest extends PlanTest {
* Evaluates a dataset to make sure that the result of calling collect matches the given
* expected answer, after sort.
*/
- protected def checkDatasetUnorderly[T : Ordering](
+ protected def checkDatasetUnorderly[T: Ordering](
ds: => Dataset[T],
expectedAnswer: T*): Unit = {
val result = getResult(ds)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 68f61cfab6d2f..909869f030312 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -1466,7 +1466,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
import org.apache.spark.unsafe.types.CalendarInterval
val df = sql("select interval 3 years -3 month 7 week 123 microseconds")
- checkAnswer(df, Row(new CalendarInterval(12 * 3 - 3, 7L * 1000 * 1000 * 3600 * 24 * 7 + 123 )))
+ checkAnswer(df, Row(new CalendarInterval(12 * 3 - 3, 7L * 1000 * 1000 * 3600 * 24 * 7 + 123)))
withTempPath(f => {
// Currently we don't yet support saving out values of interval data type.
val e = intercept[AnalysisException] {
@@ -2549,7 +2549,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
withTable("_tbl") {
sql("CREATE TABLE `_tbl`(i INT) USING parquet")
sql("INSERT INTO `_tbl` VALUES (1), (2), (3)")
- checkAnswer( sql("SELECT * FROM `_tbl`"), Row(1) :: Row(2) :: Row(3) :: Nil)
+ checkAnswer(sql("SELECT * FROM `_tbl`"), Row(1) :: Row(2) :: Row(3) :: Nil)
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
index c01666770720c..aa5cba6cdd5e9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
@@ -129,7 +129,7 @@ class SessionStateSuite extends SparkFunSuite
test("fork new session and inherit listener manager") {
class CommandCollector extends QueryExecutionListener {
val commands: ArrayBuffer[String] = ArrayBuffer.empty[String]
- override def onFailure(funcName: String, qe: QueryExecution, ex: Exception) : Unit = {}
+ override def onFailure(funcName: String, qe: QueryExecution, ex: Exception): Unit = {}
override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
commands += funcName
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
index b031c52dad8b5..b75f3840c009a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
@@ -104,7 +104,7 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
test("column stats round trip serialization") {
// Make sure we serialize and then deserialize and we will get the result data
- val df = data.toDF(stats.keys.toSeq :+ "carray" : _*)
+ val df = data.toDF(stats.keys.toSeq:+ "carray": _*)
stats.zip(df.schema).foreach { case ((k, v), field) =>
withClue(s"column $k with type ${field.dataType}") {
val roundtrip = ColumnStat.fromMap("table_is_foo", field, v.toMap(k, field.dataType))
@@ -116,7 +116,7 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
test("analyze column command - result verification") {
// (data.head.productArity - 1) because the last column does not support stats collection.
assert(stats.size == data.head.productArity - 1)
- val df = data.toDF(stats.keys.toSeq :+ "carray" : _*)
+ val df = data.toDF(stats.keys.toSeq:+ "carray": _*)
checkColStats(df, stats)
}
@@ -147,7 +147,7 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
BigInt(4444444444L) -> ("4.1 GB", "4.44E+9"),
BigInt(5555555555555L) -> ("5.1 TB", "5.56E+12"),
BigInt(6666666666666666L) -> ("5.9 PB", "6.67E+15"),
- BigInt(1L << 10 ) * (1L << 60) -> ("1024.0 EB", "1.18E+21"),
+ BigInt(1L << 10) * (1L << 60) -> ("1024.0 EB", "1.18E+21"),
BigInt(1L << 11) * (1L << 60) -> ("2.36E+21 B", "2.36E+21")
)
numbers.foreach { case (input, (expectedSize, expectedRows)) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala
index 6abcb1f067968..8829580e42ac7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReferenceSort.scala
@@ -40,7 +40,7 @@ case class ReferenceSort(
if (global) OrderedDistribution(sortOrder) :: Nil else UnspecifiedDistribution :: Nil
protected override def doExecute(): RDD[InternalRow] = attachTree(this, "sort") {
- child.execute().mapPartitions( { iterator =>
+ child.execute().mapPartitions({ iterator =>
val ordering = newOrdering(sortOrder, child.output)
val sorter = new ExternalSorter[InternalRow, Null, InternalRow](
TaskContext.get(), ordering = Some(ordering))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BenchmarkWideTable.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BenchmarkWideTable.scala
index 9dcaca0ca93ee..65314834bcbe7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BenchmarkWideTable.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/BenchmarkWideTable.scala
@@ -35,7 +35,7 @@ class BenchmarkWideTable extends BenchmarkBase {
val columns = (0 until 400).map{ i => s"id as id$i"}
val benchmark = new Benchmark("projection on wide table", N)
benchmark.addCase("wide table", numIters = 5) { iter =>
- df.selectExpr(columns : _*).queryExecution.toRdd.count()
+ df.selectExpr(columns: _*).queryExecution.toRdd.count()
}
benchmark.run()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
index 6c7779b5790d0..f118f06920ace 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
@@ -34,7 +34,7 @@ import org.apache.spark.util.Benchmark
*/
class UnsafeArrayDataBenchmark extends BenchmarkBase {
- def calculateHeaderPortionInBytes(count: Int) : Int = {
+ def calculateHeaderPortionInBytes(count: Int): Int = {
/* 4 + 4 * count // Use this expression for SPARK-15962 */
UnsafeArrayData.calculateHeaderPortionInBytes(count)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 94a2f9a00b3f3..1431e977c209a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -77,7 +77,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
/**
* Writes `data` to a Parquet file, reads it back and check file contents.
*/
- protected def checkParquetFile[T <: Product : ClassTag: TypeTag](data: Seq[T]): Unit = {
+ protected def checkParquetFile[T <: Product: ClassTag: TypeTag](data: Seq[T]): Unit = {
withParquetDataFrame(data)(r => checkAnswer(r, data.map(Row.fromTuple)))
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
index 2efff3f57d7d3..4efdc78429df1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -756,7 +756,7 @@ class ParquetQuerySuite extends QueryTest with ParquetTest with SharedSQLContext
withSQLConf(SQLConf.WHOLESTAGE_MAX_NUM_FIELDS.key -> "10") {
withTempPath { dir =>
val path = dir.getCanonicalPath
- val df = spark.range(10).select(Seq.tabulate(11) {i => ('id + i).as(s"c$i")} : _*)
+ val df = spark.range(10).select(Seq.tabulate(11) {i => ('id + i).as(s"c$i")}: _*)
df.write.mode(SaveMode.Overwrite).parquet(path)
// donot return batch, because whole stage codegen is disabled for wide table (>200 columns)
@@ -767,10 +767,10 @@ class ParquetQuerySuite extends QueryTest with ParquetTest with SharedSQLContext
// return batch
val columns = Seq.tabulate(9) {i => s"c$i"}
- val df3 = df2.selectExpr(columns : _*)
+ val df3 = df2.selectExpr(columns: _*)
val fileScan3 = df3.queryExecution.sparkPlan.find(_.isInstanceOf[FileSourceScanExec]).get
assert(fileScan3.asInstanceOf[FileSourceScanExec].supportsBatch)
- checkAnswer(df3, df.selectExpr(columns : _*))
+ checkAnswer(df3, df.selectExpr(columns: _*))
}
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index afb8ced53e25c..d3e48d9f35256 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -126,7 +126,7 @@ class BroadcastJoinSuite extends QueryTest with SQLTestUtils {
}
}
- private def assertBroadcastJoin(df : Dataset[Row]) : Unit = {
+ private def assertBroadcastJoin(df: Dataset[Row]): Unit = {
val df1 = spark.createDataFrame(Seq((1, "4"), (2, "2"))).toDF("key", "value")
val joined = df1.join(df, Seq("key"), "inner")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala
index c2087ec219e57..501656deae635 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala
@@ -160,7 +160,7 @@ class StateStoreSuite extends StateStoreSuiteBase[HDFSBackedStateStoreProvider]
store.commit()
provider.doMaintenance() // do cleanup
}
- val snapshotVersion = (0 to 10).find( version =>
+ val snapshotVersion = (0 to 10).find(version =>
fileExists(provider, version, isSnapshot = true)).getOrElse(fail("snapshot file not found"))
// Corrupt snapshot file and verify that it throws error
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index d1daf860fdfff..1c09d92d441be 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -48,7 +48,7 @@ class JDBCSuite extends SparkFunSuite
val testBytes = Array[Byte](99.toByte, 134.toByte, 135.toByte, 200.toByte, 205.toByte)
val testH2Dialect = new JdbcDialect {
- override def canHandle(url: String) : Boolean = url.startsWith("jdbc:h2")
+ override def canHandle(url: String): Boolean = url.startsWith("jdbc:h2")
override def getCatalystType(
sqlType: Int, typeName: String, size: Int, md: MetadataBuilder): Option[DataType] =
Some(StringType)
@@ -694,7 +694,7 @@ class JDBCSuite extends SparkFunSuite
test("Aggregated dialects") {
val agg = new AggregatedDialect(List(new JdbcDialect {
- override def canHandle(url: String) : Boolean = url.startsWith("jdbc:h2:")
+ override def canHandle(url: String): Boolean = url.startsWith("jdbc:h2:")
override def getCatalystType(
sqlType: Int, typeName: String, size: Int, md: MetadataBuilder): Option[DataType] =
if (sqlType % 2 == 0) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala
index bf1fd160704fa..af55eacfe26bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCWriteSuite.scala
@@ -44,7 +44,7 @@ class JDBCWriteSuite extends SharedSQLContext with BeforeAndAfter {
properties.setProperty("rowId", "false")
val testH2Dialect = new JdbcDialect {
- override def canHandle(url: String) : Boolean = url.startsWith("jdbc:h2")
+ override def canHandle(url: String): Boolean = url.startsWith("jdbc:h2")
override def isCascadingTruncateTable(): Option[Boolean] = Some(false)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
index 2108b118bf059..1b7a5cff9e201 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
@@ -1232,7 +1232,9 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
f2.setLastModified(f1.lastModified + 3600 * 1000 /* 1 hour later */)
runTwoBatchesAndVerifyResults(src, latestFirst = true, firstBatch = "2", secondBatch = "1",
+ // scalastyle:off
maxFileAge = Some("1m") /* 1 minute */)
+ // scalastyle:on
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
index b2c42eef88f6d..c2a4b0a8be485 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
@@ -120,7 +120,7 @@ trait StreamTest extends QueryTest with SharedSQLContext with Timeouts with Befo
* This operation automatically blocks until all added data has been processed.
*/
object CheckAnswer {
- def apply[A : Encoder](data: A*): CheckAnswerRows = {
+ def apply[A: Encoder](data: A*): CheckAnswerRows = {
val encoder = encoderFor[A]
val toExternalRow = RowEncoder(encoder.schema).resolveAndBind()
CheckAnswerRows(
@@ -137,7 +137,7 @@ trait StreamTest extends QueryTest with SharedSQLContext with Timeouts with Befo
* This operation automatically blocks until all added data has been processed.
*/
object CheckLastBatch {
- def apply[A : Encoder](data: A*): CheckAnswerRows = {
+ def apply[A: Encoder](data: A*): CheckAnswerRows = {
apply(isSorted = false, data: _*)
}
@@ -179,7 +179,7 @@ trait StreamTest extends QueryTest with SharedSQLContext with Timeouts with Befo
* UncaughtExceptionHandler.
* @param assertFailure a function to verify the error.
*/
- case class ExpectFailure[T <: Throwable : ClassTag](
+ case class ExpectFailure[T <: Throwable: ClassTag](
assertFailure: Throwable => Unit = _ => {},
isFatalError: Boolean = false) extends StreamAction {
val causeClass: Class[T] = implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]]
@@ -195,8 +195,8 @@ trait StreamTest extends QueryTest with SharedSQLContext with Timeouts with Befo
object Assert {
def apply(condition: => Boolean, message: String = ""): Assert = new Assert(condition, message)
- def apply(message: String)(body: => Unit): Assert = new Assert( { body; true }, message)
- def apply(body: => Unit): Assert = new Assert( { body; true }, "")
+ def apply(message: String)(body: => Unit): Assert = new Assert({ body; true }, message)
+ def apply(body: => Unit): Assert = new Assert({ body; true }, "")
}
/** Assert that a condition on the active query is true */
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala
index 599294dfbb7d7..d1d8a68f6d196 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ReflectionUtils.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.hive.thriftserver
private[hive] object ReflectionUtils {
- def setSuperField(obj : Object, fieldName: String, fieldValue: Object) {
+ def setSuperField(obj: Object, fieldName: String, fieldValue: Object) {
setAncestorField(obj, 1, fieldName, fieldValue)
}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 33e18a8da60fb..b57347fd4550d 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -327,7 +327,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
// scalastyle:off println
if (proc.isInstanceOf[Driver] || proc.isInstanceOf[SetProcessor] ||
proc.isInstanceOf[AddResourceProcessor] || proc.isInstanceOf[ListResourceProcessor] ||
- proc.isInstanceOf[ResetProcessor] ) {
+ proc.isInstanceOf[ResetProcessor]) {
val driver = new SparkSQLDriver
driver.init()
@@ -345,7 +345,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
if (ret != 0) {
// For analysis exception, only the error is printed out to the console.
rc.getException() match {
- case e : AnalysisException =>
+ case e: AnalysisException =>
err.println(s"""Error in query: ${e.getMessage}""")
case _ => err.println(rc.getErrorMessage())
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 4dec2f71b8a50..9a3fd5b8e0e4e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -784,8 +784,8 @@ private[hive] trait HiveInspectors {
case DecimalType() => PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector
case StructType(fields) =>
ObjectInspectorFactory.getStandardStructObjectInspector(
- java.util.Arrays.asList(fields.map(f => f.name) : _*),
- java.util.Arrays.asList(fields.map(f => toInspector(f.dataType)) : _*))
+ java.util.Arrays.asList(fields.map(f => f.name): _*),
+ java.util.Arrays.asList(fields.map(f => toInspector(f.dataType)): _*))
}
/**
@@ -1030,8 +1030,8 @@ private[hive] trait HiveInspectors {
getListTypeInfo(elemType.toTypeInfo)
case StructType(fields) =>
getStructTypeInfo(
- java.util.Arrays.asList(fields.map(_.name) : _*),
- java.util.Arrays.asList(fields.map(_.dataType.toTypeInfo) : _*))
+ java.util.Arrays.asList(fields.map(_.name): _*),
+ java.util.Arrays.asList(fields.map(_.dataType.toTypeInfo): _*))
case MapType(keyType, valueType, _) =>
getMapTypeInfo(keyType.toTypeInfo, valueType.toTypeInfo)
case BinaryType => binaryTypeInfo
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index be024adac8eb0..039b4546bf972 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -365,7 +365,9 @@ private[hive] class HiveClientImpl(
}
override def tableExists(dbName: String, tableName: String): Boolean = withHiveState {
- Option(client.getTable(dbName, tableName, false /* do not throw exception */)).nonEmpty
+ // scalastyle:off
+ Option(client.getTable(dbName, tableName, false/* do not throw exception */)).nonEmpty
+ // scalastyle:on
}
override def getTableOption(
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index a83ad61b204ad..34c76d5aefefd 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -91,7 +91,7 @@ private[hive] case class HiveSimpleUDF(
val ret = FunctionRegistry.invoke(
method,
function,
- conversionHelper.convertIfNecessary(inputs : _*): _*)
+ conversionHelper.convertIfNecessary(inputs: _*): _*)
unwrapper(ret)
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 4e1792321c89b..de06b8625bd92 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -248,7 +248,7 @@ private[hive] class TestHiveSparkSession(
new File(Thread.currentThread().getContextClassLoader.getResource(path).getFile)
}
- private def quoteHiveFile(path : String) = if (Utils.isWindows) {
+ private def quoteHiveFile(path: String) = if (Utils.isWindows) {
getHiveFile(path).getPath.replace('\\', '/')
} else {
getHiveFile(path).getPath
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 3de1f4aeb74dc..e44649ff4951a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -125,8 +125,8 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
case DecimalType() => PrimitiveObjectInspectorFactory.writableHiveDecimalObjectInspector
case StructType(fields) =>
ObjectInspectorFactory.getStandardStructObjectInspector(
- java.util.Arrays.asList(fields.map(f => f.name) : _*),
- java.util.Arrays.asList(fields.map(f => toWritableInspector(f.dataType)) : _*))
+ java.util.Arrays.asList(fields.map(f => f.name): _*),
+ java.util.Arrays.asList(fields.map(f => toWritableInspector(f.dataType)): _*))
}
def checkDataType(dt1: Seq[DataType], dt2: Seq[DataType]): Unit = {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
index 50506197b3138..8887a92d6d613 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
@@ -241,7 +241,9 @@ class PartitionedTablePerfStatsSuite
genericTest("file status cache respects size limit") { spec =>
withSQLConf(
SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "true",
+ // scalastyle:off
SQLConf.HIVE_FILESOURCE_PARTITION_FILE_CACHE_SIZE.key -> "1" /* 1 byte */) {
+ // scalastyle:on
withTable("test") {
withTempDir { dir =>
spec.setupTable("test", dir)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index 5fd266c2d033c..e29c6cdd43ab0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -612,7 +612,7 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
val tableName = "column_stats_test2"
// (data.head.productArity - 1) because the last column does not support stats collection.
assert(stats.size == data.head.productArity - 1)
- val df = data.toDF(stats.keys.toSeq :+ "carray" : _*)
+ val df = data.toDF(stats.keys.toSeq:+ "carray": _*)
withTable(tableName) {
df.write.saveAsTable(tableName)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 31fa3d2447467..7ef7c50b3904c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -956,7 +956,7 @@ class HiveDDLSuite
}
}
- private def withCreateTableLikeTempView(location : Option[String]): Unit = {
+ private def withCreateTableLikeTempView(location: Option[String]): Unit = {
val sourceViewName = "tab1"
val targetTabName = "tab2"
val tableType = if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED
@@ -988,7 +988,7 @@ class HiveDDLSuite
}
}
- private def withCreateTableLikeDSTable(location : Option[String]): Unit = {
+ private def withCreateTableLikeDSTable(location: Option[String]): Unit = {
val sourceTabName = "tab1"
val targetTabName = "tab2"
val tableType = if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED
@@ -1023,7 +1023,7 @@ class HiveDDLSuite
}
}
- private def withCreateTableLikeExtDSTable(location : Option[String]): Unit = {
+ private def withCreateTableLikeExtDSTable(location: Option[String]): Unit = {
val sourceTabName = "tab1"
val targetTabName = "tab2"
val tableType = if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED
@@ -1061,7 +1061,7 @@ class HiveDDLSuite
}
}
- private def withCreateTableLikeManagedHiveTable(location : Option[String]): Unit = {
+ private def withCreateTableLikeManagedHiveTable(location: Option[String]): Unit = {
val sourceTabName = "tab1"
val targetTabName = "tab2"
val tableType = if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED
@@ -1093,7 +1093,7 @@ class HiveDDLSuite
}
}
- private def withCreateTableLikeExtHiveTable(location : Option[String]): Unit = {
+ private def withCreateTableLikeExtHiveTable(location: Option[String]): Unit = {
val catalog = spark.sessionState.catalog
val tableType = if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED
withTempDir { tmpDir =>
@@ -1143,7 +1143,7 @@ class HiveDDLSuite
}
}
- private def withCreateTableLikeView(location : Option[String]): Unit = {
+ private def withCreateTableLikeView(location: Option[String]): Unit = {
val sourceTabName = "tab1"
val sourceViewName = "view"
val targetTabName = "tab2"
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index cf33760360724..08f87ea194200 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -265,7 +265,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
// Jdk version leads to different query output for double, so not use createQueryTest here
test("division") {
val res = sql("SELECT 2 / 1, 1 / 2, 1 / 3, 1 / COUNT(*) FROM src LIMIT 1").collect().head
- Seq(2.0, 0.5, 0.3333333333333333, 0.002).zip(res.toSeq).foreach( x =>
+ Seq(2.0, 0.5, 0.3333333333333333, 0.002).zip(res.toSeq).foreach(x =>
assert(x._1 == x._2.asInstanceOf[Double]))
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
index 23f21e6b9931e..6cd8120f0d9b6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
@@ -691,7 +691,7 @@ class ParquetSourceSuite extends ParquetPartitioningTest {
"partitioned_parquet_with_key_and_complextypes",
"normal_parquet")
- sql( s"""
+ sql(s"""
CREATE TEMPORARY VIEW partitioned_parquet
USING org.apache.spark.sql.parquet
OPTIONS (
@@ -699,7 +699,7 @@ class ParquetSourceSuite extends ParquetPartitioningTest {
)
""")
- sql( s"""
+ sql(s"""
CREATE TEMPORARY VIEW partitioned_parquet_with_key
USING org.apache.spark.sql.parquet
OPTIONS (
@@ -707,7 +707,7 @@ class ParquetSourceSuite extends ParquetPartitioningTest {
)
""")
- sql( s"""
+ sql(s"""
CREATE TEMPORARY VIEW normal_parquet
USING org.apache.spark.sql.parquet
OPTIONS (
@@ -715,7 +715,7 @@ class ParquetSourceSuite extends ParquetPartitioningTest {
)
""")
- sql( s"""
+ sql(s"""
CREATE TEMPORARY VIEW partitioned_parquet_with_key_and_complextypes
USING org.apache.spark.sql.parquet
OPTIONS (
@@ -723,7 +723,7 @@ class ParquetSourceSuite extends ParquetPartitioningTest {
)
""")
- sql( s"""
+ sql(s"""
CREATE TEMPORARY VIEW partitioned_parquet_with_complextypes
USING org.apache.spark.sql.parquet
OPTIONS (
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/BasicOperationsSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/BasicOperationsSuite.scala
index a3062ac94614b..717b7c0960598 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/BasicOperationsSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/BasicOperationsSuite.scala
@@ -64,9 +64,9 @@ class BasicOperationsSuite extends TestSuiteBase {
assert(numInputPartitions === 2, "Number of input partitions has been changed from 2")
val input = Seq(1 to 4, 5 to 8, 9 to 12)
val output = Seq(
- Seq( Seq(1, 2), Seq(3, 4) ),
- Seq( Seq(5, 6), Seq(7, 8) ),
- Seq( Seq(9, 10), Seq(11, 12) )
+ Seq(Seq(1, 2), Seq(3, 4)),
+ Seq(Seq(5, 6), Seq(7, 8)),
+ Seq(Seq(9, 10), Seq(11, 12))
)
val operation = (r: DStream[Int]) => r.glom().map(_.toSeq)
testOperation(input, operation, output)
@@ -124,18 +124,18 @@ class BasicOperationsSuite extends TestSuiteBase {
test("groupByKey") {
testOperation(
- Seq( Seq("a", "a", "b"), Seq("", ""), Seq() ),
+ Seq(Seq("a", "a", "b"), Seq("", ""), Seq()),
(s: DStream[String]) => s.map(x => (x, 1)).groupByKey().mapValues(_.toSeq),
- Seq( Seq(("a", Seq(1, 1)), ("b", Seq(1))), Seq(("", Seq(1, 1))), Seq() ),
+ Seq(Seq(("a", Seq(1, 1)), ("b", Seq(1))), Seq(("", Seq(1, 1))), Seq()),
true
)
}
test("reduceByKey") {
testOperation(
- Seq( Seq("a", "a", "b"), Seq("", ""), Seq() ),
+ Seq(Seq("a", "a", "b"), Seq("", ""), Seq()),
(s: DStream[String]) => s.map(x => (x, 1)).reduceByKey(_ + _),
- Seq( Seq(("a", 2), ("b", 1)), Seq(("", 2)), Seq() ),
+ Seq(Seq(("a", 2), ("b", 1)), Seq(("", 2)), Seq()),
true
)
}
@@ -167,20 +167,20 @@ class BasicOperationsSuite extends TestSuiteBase {
test("mapValues") {
testOperation(
- Seq( Seq("a", "a", "b"), Seq("", ""), Seq() ),
+ Seq(Seq("a", "a", "b"), Seq("", ""), Seq()),
(s: DStream[String]) => s.map(x => (x, 1)).reduceByKey(_ + _).mapValues(_ + 10),
- Seq( Seq(("a", 12), ("b", 11)), Seq(("", 12)), Seq() ),
+ Seq(Seq(("a", 12), ("b", 11)), Seq(("", 12)), Seq()),
true
)
}
test("flatMapValues") {
testOperation(
- Seq( Seq("a", "a", "b"), Seq("", ""), Seq() ),
+ Seq(Seq("a", "a", "b"), Seq("", ""), Seq()),
(s: DStream[String]) => {
s.map(x => (x, 1)).reduceByKey(_ + _).flatMapValues(x => Seq(x, x + 10))
},
- Seq( Seq(("a", 2), ("a", 12), ("b", 1), ("b", 11)), Seq(("", 2), ("", 12)), Seq() ),
+ Seq(Seq(("a", 2), ("a", 12), ("b", 1), ("b", 11)), Seq(("", 2), ("", 12)), Seq()),
true
)
}
@@ -256,13 +256,13 @@ class BasicOperationsSuite extends TestSuiteBase {
}
test("transformWith") {
- val inputData1 = Seq( Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
- val inputData2 = Seq( Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
+ val inputData1 = Seq(Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
+ val inputData2 = Seq(Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
val outputData = Seq(
- Seq( ("a", (1, "x")), ("b", (1, "x")) ),
- Seq( ("", (1, "x")) ),
- Seq( ),
- Seq( )
+ Seq(("a", (1, "x")), ("b", (1, "x"))),
+ Seq(("", (1, "x"))),
+ Seq(),
+ Seq()
)
val operation = (s1: DStream[String], s2: DStream[String]) => {
val t1 = s1.map(x => (x, 1))
@@ -276,8 +276,8 @@ class BasicOperationsSuite extends TestSuiteBase {
}
test("transformWith with input stream return None") {
- val inputData1 = Seq( Seq("a", "b"), Seq("a", ""), Seq(""), null )
- val inputData2 = Seq( Seq("a", "b"), Seq("b", ""), Seq(), null )
+ val inputData1 = Seq(Seq("a", "b"), Seq("a", ""), Seq(""), null )
+ val inputData2 = Seq(Seq("a", "b"), Seq("b", ""), Seq(), null )
val outputData = Seq(
Seq("a", "b", "a", "b"),
Seq("a", "b", "", ""),
@@ -331,13 +331,13 @@ class BasicOperationsSuite extends TestSuiteBase {
}
test("cogroup") {
- val inputData1 = Seq( Seq("a", "a", "b"), Seq("a", ""), Seq(""), Seq() )
- val inputData2 = Seq( Seq("a", "a", "b"), Seq("b", ""), Seq(), Seq() )
+ val inputData1 = Seq(Seq("a", "a", "b"), Seq("a", ""), Seq(""), Seq() )
+ val inputData2 = Seq(Seq("a", "a", "b"), Seq("b", ""), Seq(), Seq() )
val outputData = Seq(
- Seq( ("a", (Seq(1, 1), Seq("x", "x"))), ("b", (Seq(1), Seq("x"))) ),
- Seq( ("a", (Seq(1), Seq())), ("b", (Seq(), Seq("x"))), ("", (Seq(1), Seq("x"))) ),
- Seq( ("", (Seq(1), Seq())) ),
- Seq( )
+ Seq(("a", (Seq(1, 1), Seq("x", "x"))), ("b", (Seq(1), Seq("x")))),
+ Seq(("a", (Seq(1), Seq())), ("b", (Seq(), Seq("x"))), ("", (Seq(1), Seq("x")))),
+ Seq(("", (Seq(1), Seq()))),
+ Seq()
)
val operation = (s1: DStream[String], s2: DStream[String]) => {
s1.map(x => (x, 1)).cogroup(s2.map(x => (x, "x"))).mapValues(x => (x._1.toSeq, x._2.toSeq))
@@ -346,13 +346,13 @@ class BasicOperationsSuite extends TestSuiteBase {
}
test("join") {
- val inputData1 = Seq( Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
- val inputData2 = Seq( Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
+ val inputData1 = Seq(Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
+ val inputData2 = Seq(Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
val outputData = Seq(
- Seq( ("a", (1, "x")), ("b", (1, "x")) ),
- Seq( ("", (1, "x")) ),
- Seq( ),
- Seq( )
+ Seq(("a", (1, "x")), ("b", (1, "x"))),
+ Seq(("", (1, "x"))),
+ Seq(),
+ Seq()
)
val operation = (s1: DStream[String], s2: DStream[String]) => {
s1.map(x => (x, 1)).join(s2.map(x => (x, "x")))
@@ -361,13 +361,13 @@ class BasicOperationsSuite extends TestSuiteBase {
}
test("leftOuterJoin") {
- val inputData1 = Seq( Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
- val inputData2 = Seq( Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
+ val inputData1 = Seq(Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
+ val inputData2 = Seq(Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
val outputData = Seq(
- Seq( ("a", (1, Some("x"))), ("b", (1, Some("x"))) ),
- Seq( ("", (1, Some("x"))), ("a", (1, None)) ),
- Seq( ("", (1, None)) ),
- Seq( )
+ Seq(("a", (1, Some("x"))), ("b", (1, Some("x")))),
+ Seq(("", (1, Some("x"))), ("a", (1, None))),
+ Seq(("", (1, None))),
+ Seq()
)
val operation = (s1: DStream[String], s2: DStream[String]) => {
s1.map(x => (x, 1)).leftOuterJoin(s2.map(x => (x, "x")))
@@ -376,13 +376,13 @@ class BasicOperationsSuite extends TestSuiteBase {
}
test("rightOuterJoin") {
- val inputData1 = Seq( Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
- val inputData2 = Seq( Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
+ val inputData1 = Seq(Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
+ val inputData2 = Seq(Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
val outputData = Seq(
- Seq( ("a", (Some(1), "x")), ("b", (Some(1), "x")) ),
- Seq( ("", (Some(1), "x")), ("b", (None, "x")) ),
- Seq( ),
- Seq( ("", (None, "x")) )
+ Seq(("a", (Some(1), "x")), ("b", (Some(1), "x"))),
+ Seq(("", (Some(1), "x")), ("b", (None, "x"))),
+ Seq(),
+ Seq(("", (None, "x")))
)
val operation = (s1: DStream[String], s2: DStream[String]) => {
s1.map(x => (x, 1)).rightOuterJoin(s2.map(x => (x, "x")))
@@ -391,13 +391,13 @@ class BasicOperationsSuite extends TestSuiteBase {
}
test("fullOuterJoin") {
- val inputData1 = Seq( Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
- val inputData2 = Seq( Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
+ val inputData1 = Seq(Seq("a", "b"), Seq("a", ""), Seq(""), Seq() )
+ val inputData2 = Seq(Seq("a", "b"), Seq("b", ""), Seq(), Seq("") )
val outputData = Seq(
- Seq( ("a", (Some(1), Some("x"))), ("b", (Some(1), Some("x"))) ),
- Seq( ("", (Some(1), Some("x"))), ("a", (Some(1), None)), ("b", (None, Some("x"))) ),
- Seq( ("", (Some(1), None)) ),
- Seq( ("", (None, Some("x"))) )
+ Seq(("a", (Some(1), Some("x"))), ("b", (Some(1), Some("x")))),
+ Seq(("", (Some(1), Some("x"))), ("a", (Some(1), None)), ("b", (None, Some("x")))),
+ Seq(("", (Some(1), None))),
+ Seq(("", (None, Some("x"))))
)
val operation = (s1: DStream[String], s2: DStream[String]) => {
s1.map(x => (x, 1)).fullOuterJoin(s2.map(x => (x, "x")))
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/CheckpointSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/CheckpointSuite.scala
index ee2fd45a7e851..c28377816617e 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/CheckpointSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/CheckpointSuite.scala
@@ -415,7 +415,7 @@ class CheckpointSuite extends TestSuiteBase with DStreamCheckpointTester
// source - TestInputDStream.
test("recovery with map and reduceByKey operations") {
testCheckpointedOperation(
- Seq( Seq("a", "a", "b"), Seq("", ""), Seq(), Seq("a", "a", "b"), Seq("", ""), Seq() ),
+ Seq(Seq("a", "a", "b"), Seq("", ""), Seq(), Seq("a", "a", "b"), Seq("", ""), Seq()),
(s: DStream[String]) => s.map(x => (x, 1)).reduceByKey(_ + _),
Seq(
Seq(("a", 2), ("b", 1)),
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala
index fe65353b9d502..933ec126d525b 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ReceivedBlockHandlerSuite.scala
@@ -100,7 +100,7 @@ abstract class BaseReceivedBlockHandlerSuite(enableEncryption: Boolean)
}
after {
- for ( blockManager <- blockManagerBuffer ) {
+ for (blockManager <- blockManagerBuffer) {
if (blockManager != null) {
blockManager.stop()
}
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
index e7cec999c219e..7a1429d0eb4e3 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
@@ -93,7 +93,7 @@ class UISeleniumSuite
eventually(timeout(10 seconds), interval(50 milliseconds)) {
go to (sparkUI.webUrl.stripSuffix("/"))
- find(cssSelector( """ul li a[href*="streaming"]""")) should not be (None)
+ find(cssSelector("""ul li a[href*="streaming"]""")) should not be (None)
}
eventually(timeout(10 seconds), interval(50 milliseconds)) {
@@ -154,7 +154,7 @@ class UISeleniumSuite
outputOpIds.map(_.text) should be (List("0", "1"))
// Check job ids
- val jobIdCells = findAll(cssSelector( """#batch-job-table a""")).toSeq
+ val jobIdCells = findAll(cssSelector("""#batch-job-table a""")).toSeq
jobIdCells.map(_.text) should be (List("0", "1", "2", "3"))
val jobLinks = jobIdCells.flatMap(_.attribute("href"))
@@ -192,7 +192,7 @@ class UISeleniumSuite
eventually(timeout(10 seconds), interval(50 milliseconds)) {
go to (sparkUI.webUrl.stripSuffix("/"))
- find(cssSelector( """ul li a[href*="streaming"]""")) should be(None)
+ find(cssSelector("""ul li a[href*="streaming"]""")) should be(None)
}
eventually(timeout(10 seconds), interval(50 milliseconds)) {
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/WindowOperationsSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/WindowOperationsSuite.scala
index c7d085ec0799b..7b529f3e64ef8 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/WindowOperationsSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/WindowOperationsSuite.scala
@@ -116,36 +116,36 @@ class WindowOperationsSuite extends TestSuiteBase {
testWindow(
"basic window",
- Seq( Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
- Seq( Seq(0), Seq(0, 1), Seq(1, 2), Seq(2, 3), Seq(3, 4), Seq(4, 5))
+ Seq(Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
+ Seq(Seq(0), Seq(0, 1), Seq(1, 2), Seq(2, 3), Seq(3, 4), Seq(4, 5))
)
testWindow(
"tumbling window",
- Seq( Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
- Seq( Seq(0, 1), Seq(2, 3), Seq(4, 5)),
+ Seq(Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
+ Seq(Seq(0, 1), Seq(2, 3), Seq(4, 5)),
Seconds(2),
Seconds(2)
)
testWindow(
"larger window",
- Seq( Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
- Seq( Seq(0, 1), Seq(0, 1, 2, 3), Seq(2, 3, 4, 5), Seq(4, 5)),
+ Seq(Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
+ Seq(Seq(0, 1), Seq(0, 1, 2, 3), Seq(2, 3, 4, 5), Seq(4, 5)),
Seconds(4),
Seconds(2)
)
testWindow(
"non-overlapping window",
- Seq( Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
- Seq( Seq(1, 2), Seq(4, 5)),
+ Seq(Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5)),
+ Seq(Seq(1, 2), Seq(4, 5)),
Seconds(2),
Seconds(3)
)
test("window - persistence level") {
- val input = Seq( Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5))
+ val input = Seq(Seq(0), Seq(1), Seq(2), Seq(3), Seq(4), Seq(5))
val ssc = new StreamingContext(conf, batchDuration)
val inputStream = new TestInputStream[Int](ssc, input, 1)
val windowStream1 = inputStream.window(batchDuration * 2)
@@ -161,26 +161,26 @@ class WindowOperationsSuite extends TestSuiteBase {
testReduceByKeyAndWindow(
"basic reduction",
- Seq( Seq(("a", 1), ("a", 3)) ),
- Seq( Seq(("a", 4)) )
+ Seq(Seq(("a", 1), ("a", 3))),
+ Seq(Seq(("a", 4)))
)
testReduceByKeyAndWindow(
"key already in window and new value added into window",
- Seq( Seq(("a", 1)), Seq(("a", 1)) ),
- Seq( Seq(("a", 1)), Seq(("a", 2)) )
+ Seq(Seq(("a", 1)), Seq(("a", 1))),
+ Seq(Seq(("a", 1)), Seq(("a", 2)))
)
testReduceByKeyAndWindow(
"new key added into window",
- Seq( Seq(("a", 1)), Seq(("a", 1), ("b", 1)) ),
- Seq( Seq(("a", 1)), Seq(("a", 2), ("b", 1)) )
+ Seq(Seq(("a", 1)), Seq(("a", 1), ("b", 1))),
+ Seq(Seq(("a", 1)), Seq(("a", 2), ("b", 1)))
)
testReduceByKeyAndWindow(
"key removed from window",
- Seq( Seq(("a", 1)), Seq(("a", 1)), Seq(), Seq() ),
- Seq( Seq(("a", 1)), Seq(("a", 2)), Seq(("a", 1)), Seq() )
+ Seq(Seq(("a", 1)), Seq(("a", 1)), Seq(), Seq()),
+ Seq(Seq(("a", 1)), Seq(("a", 2)), Seq(("a", 1)), Seq())
)
testReduceByKeyAndWindow(
@@ -197,26 +197,26 @@ class WindowOperationsSuite extends TestSuiteBase {
testReduceByKeyAndWindowWithInverse(
"basic reduction",
- Seq(Seq(("a", 1), ("a", 3)) ),
- Seq(Seq(("a", 4)) )
+ Seq(Seq(("a", 1), ("a", 3))),
+ Seq(Seq(("a", 4)))
)
testReduceByKeyAndWindowWithInverse(
"key already in window and new value added into window",
- Seq( Seq(("a", 1)), Seq(("a", 1)) ),
- Seq( Seq(("a", 1)), Seq(("a", 2)) )
+ Seq(Seq(("a", 1)), Seq(("a", 1))),
+ Seq(Seq(("a", 1)), Seq(("a", 2)))
)
testReduceByKeyAndWindowWithInverse(
"new key added into window",
- Seq( Seq(("a", 1)), Seq(("a", 1), ("b", 1)) ),
- Seq( Seq(("a", 1)), Seq(("a", 2), ("b", 1)) )
+ Seq(Seq(("a", 1)), Seq(("a", 1), ("b", 1))),
+ Seq(Seq(("a", 1)), Seq(("a", 2), ("b", 1)))
)
testReduceByKeyAndWindowWithInverse(
"key removed from window",
- Seq( Seq(("a", 1)), Seq(("a", 1)), Seq(), Seq() ),
- Seq( Seq(("a", 1)), Seq(("a", 2)), Seq(("a", 1)), Seq(("a", 0)) )
+ Seq(Seq(("a", 1)), Seq(("a", 1)), Seq(), Seq()),
+ Seq(Seq(("a", 1)), Seq(("a", 2)), Seq(("a", 1)), Seq(("a", 0)))
)
testReduceByKeyAndWindowWithInverse(
@@ -245,7 +245,7 @@ class WindowOperationsSuite extends TestSuiteBase {
test("countByWindow") {
val input = Seq(Seq(1), Seq(1), Seq(1, 2), Seq(0), Seq(), Seq() )
- val expectedOutput = Seq( Seq(1), Seq(2), Seq(3), Seq(3), Seq(1), Seq(0))
+ val expectedOutput = Seq(Seq(1), Seq(2), Seq(3), Seq(3), Seq(1), Seq(0))
val windowDuration = Seconds(2)
val slideDuration = Seconds(1)
val numBatches = expectedOutput.size * (slideDuration / batchDuration).toInt
@@ -257,7 +257,7 @@ class WindowOperationsSuite extends TestSuiteBase {
test("countByValueAndWindow") {
val input = Seq(Seq("a"), Seq("b", "b"), Seq("a", "b"))
- val expectedOutput = Seq( Seq(("a", 1)), Seq(("a", 1), ("b", 2)), Seq(("a", 1), ("b", 3)))
+ val expectedOutput = Seq(Seq(("a", 1)), Seq(("a", 1), ("b", 2)), Seq(("a", 1), ("b", 3)))
val windowDuration = Seconds(2)
val slideDuration = Seconds(1)
val numBatches = expectedOutput.size * (slideDuration / batchDuration).toInt