Merge branch 'master' into r-stop-paste

Author: Michael Chirico

R/pkg/R/DataFrame.R

@@ -1225,23 +1225,18 @@ setMethod("collect",
               # empty data.frame with 0 columns and 0 rows
               data.frame()
             } else if (useArrow) {
-              requireNamespace1 <- requireNamespace
-              if (requireNamespace1("arrow", quietly = TRUE)) {
-                read_arrow <- get("read_arrow", envir = asNamespace("arrow"), inherits = FALSE)
-                # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-                useAsTibble <- exists("as_tibble", envir = asNamespace("arrow"))
-
+              if (requireNamespace("arrow", quietly = TRUE)) {
                 portAuth <- callJMethod(x@sdf, "collectAsArrowToR")
                 port <- portAuth[[1]]
                 authSecret <- portAuth[[2]]
                 conn <- socketConnection(
                   port = port, blocking = TRUE, open = "wb", timeout = connectionTimeout)
                 output <- tryCatch({
                   doServerAuth(conn, authSecret)
-                  arrowTable <- read_arrow(readRaw(conn))
-                  if (useAsTibble) {
-                    as_tibble <- get("as_tibble", envir = asNamespace("arrow"))
-                    as.data.frame(as_tibble(arrowTable), stringsAsFactors = stringsAsFactors)
+                  arrowTable <- arrow::read_arrow(readRaw(conn))
+                  # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
+                  if (exists("as_tibble", envir = asNamespace("arrow"))) {
+                    as.data.frame(arrow::as_tibble(arrowTable), stringsAsFactors = stringsAsFactors)
                   } else {
                     as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
                   }

R/pkg/R/types.R

@@ -88,8 +88,7 @@ specialtypeshandle <- function(type) {
 checkSchemaInArrow <- function(schema) {
   stopifnot(inherits(schema, "structType"))
 
-  requireNamespace1 <- requireNamespace
-  if (!requireNamespace1("arrow", quietly = TRUE)) {
+  if (!requireNamespace("arrow", quietly = TRUE)) {
     stop("'arrow' package should be installed.")
   }
 

dev/appveyor-install-dependencies.ps1

@@ -50,7 +50,7 @@ Function InstallR {
 
 Function InstallRtools {
   $rtoolsver = $rToolsVer.Split('.')[0..1] -Join ''
-  $rtoolsurl = $CRAN + "/bin/windows/Rtools/Rtools$rtoolsver.exe"
+  $rtoolsurl = $CRAN + "/bin/windows/Rtools/rtools$rtoolsver-x86_64.exe"
 
   # Downloading Rtools
   Start-FileDownload $rtoolsurl "Rtools-current.exe"
@@ -67,8 +67,8 @@ Function InstallRtools {
   Else {
     $gccPath = $env:GCC_PATH
   }
-  $env:PATH = $RtoolsDrive + '\Rtools\bin;' + $RtoolsDrive + '\Rtools\MinGW\bin;' + $RtoolsDrive + '\Rtools\' + $gccPath + '\bin;' + $env:PATH
-  $env:BINPREF=$RtoolsDrive + '/Rtools/mingw_$(WIN)/bin/'
+  $env:PATH = $RtoolsDrive + '\Rtools40\bin;' + $RtoolsDrive + '\Rtools40\MinGW$(WIN)\bin;' + $RtoolsDrive + '\Rtools40\' + $gccPath + '\bin;' + $env:PATH
+  $env:BINPREF=$RtoolsDrive + '/Rtools40/mingw$(WIN)/bin/'
 }
 
 # create tools directory outside of Spark directory
@@ -116,7 +116,7 @@ Pop-Location
 
 # ========================== R
 $rVer = "3.6.2"
-$rToolsVer = "3.5.1"
+$rToolsVer = "4.0.0"
 
 InstallR
 InstallRtools

docs/_plugins/copy_api_dirs.rb

@@ -157,8 +157,8 @@
     curr_dir = pwd
     cd("..")
 
-    puts "Running 'build/sbt clean package' from " + pwd + "; this may take a few minutes..."
-    system("build/sbt clean package") || raise("SQL doc generation failed")
+    puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..."
+    system("build/sbt clean package -Phive") || raise("SQL doc generation failed")
 
     puts "Moving back into docs dir."
     cd("docs")

docs/sql-migration-guide.md

@@ -59,7 +59,7 @@ license: |
 
   - In Spark 3.0, you can use `ADD FILE` to add file directories as well. Earlier you could add only single files using this command. To restore the behavior of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
 
-  - In Spark 3.0, `SHOW TBLPROPERTIES` throws `AnalysisException` if the table does not exist. In Spark version 2.4 and below, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view causes `AnalysisException`. In Spark version 2.4 and below, it returned an empty result.
+  - In Spark 3.0, `SHOW TBLPROPERTIES` throws `AnalysisException` if the table does not exist. In Spark version 2.4 and below, this scenario caused `NoSuchTableException`.
 
   - In Spark 3.0, `SHOW CREATE TABLE` always returns Spark DDL, even when the given table is a Hive SerDe table. For generating Hive DDL, use `SHOW CREATE TABLE AS SERDE` command instead.
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala

@@ -521,7 +521,7 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
   override def sql: String = {
     val valueSQL = child.sql
     val listSQL = hset.toSeq
-      .map(elem => Literal(convertToScala(elem, child.dataType)).sql)
+      .map(elem => Literal(elem, child.dataType).sql)
       .mkString(", ")
     s"($valueSQL IN ($listSQL))"
   }

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala

@@ -83,9 +83,12 @@ trait ExpressionWithRandomSeed {
   """,
   since = "1.5.0")
 // scalastyle:on line.size.limit
-case class Rand(child: Expression) extends RDG with ExpressionWithRandomSeed {
+case class Rand(child: Expression, hideSeed: Boolean = false)
+  extends RDG with ExpressionWithRandomSeed {
 
-  def this() = this(Literal(Utils.random.nextLong(), LongType))
+  def this() = this(Literal(Utils.random.nextLong(), LongType), true)
+
+  def this(child: Expression) = this(child, false)
 
   override def withNewSeed(seed: Long): Rand = Rand(Literal(seed, LongType))
 
@@ -101,7 +104,12 @@ case class Rand(child: Expression) extends RDG with ExpressionWithRandomSeed {
       isNull = FalseLiteral)
   }
 
-  override def freshCopy(): Rand = Rand(child)
+  override def freshCopy(): Rand = Rand(child, hideSeed)
+
+  override def flatArguments: Iterator[Any] = Iterator(child)
+  override def sql: String = {
+    s"rand(${if (hideSeed) "" else child.sql})"
+  }
 }
 
 object Rand {
@@ -126,9 +134,12 @@ object Rand {
   """,
   since = "1.5.0")
 // scalastyle:on line.size.limit
-case class Randn(child: Expression) extends RDG with ExpressionWithRandomSeed {
+case class Randn(child: Expression, hideSeed: Boolean = false)
+  extends RDG with ExpressionWithRandomSeed {
 
-  def this() = this(Literal(Utils.random.nextLong(), LongType))
+  def this() = this(Literal(Utils.random.nextLong(), LongType), true)
+
+  def this(child: Expression) = this(child, false)
 
   override def withNewSeed(seed: Long): Randn = Randn(Literal(seed, LongType))
 
@@ -144,7 +155,12 @@ case class Randn(child: Expression) extends RDG with ExpressionWithRandomSeed {
       isNull = FalseLiteral)
   }
 
-  override def freshCopy(): Randn = Randn(child)
+  override def freshCopy(): Randn = Randn(child, hideSeed)
+
+  override def flatArguments: Iterator[Any] = Iterator(child)
+  override def sql: String = {
+    s"randn(${if (hideSeed) "" else child.sql})"
+  }
 }
 
 object Randn {

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala

@@ -3573,7 +3573,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   override def visitShowTblProperties(
       ctx: ShowTblPropertiesContext): LogicalPlan = withOrigin(ctx) {
     ShowTableProperties(
-      UnresolvedTable(visitMultipartIdentifier(ctx.table)),
+      UnresolvedTableOrView(visitMultipartIdentifier(ctx.table)),
       Option(ctx.key).map(visitTablePropertyKey))
   }
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala

@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.math.BigDecimal
 import java.util.concurrent.TimeUnit
 
 import scala.util.control.NonFatal
@@ -55,11 +54,12 @@ object IntervalUtils {
   }
 
   def getDays(interval: CalendarInterval): Int = {
-    interval.days
+    val daysInMicroseconds = (interval.microseconds / MICROS_PER_DAY).toInt
+    Math.addExact(interval.days, daysInMicroseconds)
   }
 
   def getHours(interval: CalendarInterval): Long = {
-    interval.microseconds / MICROS_PER_HOUR
+    (interval.microseconds % MICROS_PER_DAY) / MICROS_PER_HOUR
   }
 
   def getMinutes(interval: CalendarInterval): Byte = {

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala

@@ -68,6 +68,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Years and months must not be taken into account
     checkEvaluation(ExtractIntervalDays("100 year 10 months 5 days"), 5)
     checkEvaluation(ExtractIntervalDays(largeInterval), 31)
+    checkEvaluation(ExtractIntervalDays("25 hours"), 1)
   }
 
   test("hours") {
@@ -81,6 +82,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Minutes should be taken into account
     checkEvaluation(ExtractIntervalHours("10 hours 100 minutes"), 11L)
     checkEvaluation(ExtractIntervalHours(largeInterval), 11L)
+    checkEvaluation(ExtractIntervalHours("25 hours"), 1L)
+
   }
 
   test("minutes") {