Updated.

Author: petermaxlee

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SortOrder.scala

@@ -54,10 +54,7 @@ case object NullsLast extends NullOrdering{
  * An expression that can be used to sort a tuple.  This class extends expression primarily so that
  * transformations over expression will descend into its child.
  */
-case class SortOrder(
-  child: Expression,
-  direction: SortDirection,
-  nullOrdering: NullOrdering)
+case class SortOrder(child: Expression, direction: SortDirection, nullOrdering: NullOrdering)
   extends UnaryExpression with Unevaluable {
 
   /** Sort order is not foldable because we don't have an eval for it. */
@@ -94,34 +91,23 @@ case class SortPrefix(child: SortOrder) extends UnaryExpression {
 
   val nullValue = child.child.dataType match {
     case BooleanType | DateType | TimestampType | _: IntegralType =>
-      if (nullAsSmallest) {
-        Long.MinValue
-      } else {
-        Long.MaxValue
-      }
+      if (nullAsSmallest) Long.MinValue else Long.MaxValue
     case dt: DecimalType if dt.precision - dt.scale <= Decimal.MAX_LONG_DIGITS =>
-      if (nullAsSmallest) {
-        Long.MinValue
-      } else {
-        Long.MaxValue
-      }
+      if (nullAsSmallest) Long.MinValue else Long.MaxValue
     case _: DecimalType =>
       if (nullAsSmallest) {
         DoublePrefixComparator.computePrefix(Double.NegativeInfinity)
       } else {
         DoublePrefixComparator.computePrefix(Double.NaN)
       }
     case _ =>
-      if (nullAsSmallest) {
-        0L
-      } else {
-        -1L
-      }
+      if (nullAsSmallest) 0L else -1L
   }
 
-  private def nullAsSmallest: Boolean = (child.isAscending && child.nullOrdering == NullsFirst) ||
+  private def nullAsSmallest: Boolean = {
+    (child.isAscending && child.nullOrdering == NullsFirst) ||
       (!child.isAscending && child.nullOrdering == NullsLast)
-
+  }
 
   override def eval(input: InternalRow): Any = throw new UnsupportedOperationException
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala

@@ -100,16 +100,16 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR
             // Nothing
           } else if ($isNullA) {
             return ${
-        order.nullOrdering match {
-          case NullsFirst => "-1"
-          case NullsLast => "1"
-        }};
+              order.nullOrdering match {
+                case NullsFirst => "-1"
+                case NullsLast => "1"
+              }};
           } else if ($isNullB) {
             return ${
-        order.nullOrdering match {
-          case NullsFirst => "1"
-          case NullsLast => "-1"
-        }};
+              order.nullOrdering match {
+                case NullsFirst => "1"
+                case NullsLast => "-1"
+              }};
           } else {
             int comp = ${ctx.genComp(order.child.dataType, primitiveA, primitiveB)};
             if (comp != 0) {

sql/core/src/main/scala/org/apache/spark/sql/Column.scala

@@ -1007,7 +1007,7 @@ class Column(protected[sql] val expr: Expression) extends Logging {
   /**
    * Returns an ordering used in sorting.
    * {{{
-   *   // Scala: sort a DataFrame by age column in descending order.
+   *   // Scala
    *   df.sort(df("age").desc)
    *
    *   // Java
@@ -1020,9 +1020,9 @@ class Column(protected[sql] val expr: Expression) extends Logging {
   def desc: Column = withExpr { SortOrder(expr, Descending) }
 
   /**
-   * Returns an ordering used in sorting.
+   * Returns a descending ordering used in sorting, where null values appear before non-null values.
    * {{{
-   *   // Scala: sort a DataFrame by age column in descending order with NULLS FIRST.
+   *   // Scala: sort a DataFrame by age column in descending order and null values appearing first.
    *   df.sort(df("age").desc_nulls_first)
    *
    *   // Java
@@ -1035,7 +1035,22 @@ class Column(protected[sql] val expr: Expression) extends Logging {
   def desc_nulls_first: Column = withExpr { SortOrder(expr, Descending, NullsFirst) }
 
   /**
-   * Returns an ordering used in sorting.
+   * Returns a descending ordering used in sorting, where null values appear after non-null values.
+   * {{{
+   *   // Scala: sort a DataFrame by age column in descending order and null values appearing last.
+   *   df.sort(df("age").desc_nulls_last)
+   *
+   *   // Java
+   *   df.sort(df.col("age").desc_nulls_last());
+   * }}}
+   *
+   * @group expr_ops
+   * @since 2.1.0
+   */
+  def desc_nulls_last: Column = withExpr { SortOrder(expr, Descending, NullsLast) }
+
+  /**
+   * Returns an ascending ordering used in sorting.
    * {{{
    *   // Scala: sort a DataFrame by age column in ascending order.
    *   df.sort(df("age").asc)
@@ -1050,9 +1065,9 @@ class Column(protected[sql] val expr: Expression) extends Logging {
   def asc: Column = withExpr { SortOrder(expr, Ascending) }
 
   /**
-   * Returns an ordering used in sorting.
+   * Returns an ascending ordering used in sorting, where null values appear before non-null values.
    * {{{
-   *   // Scala: sort a DataFrame by age column in ascending order with NULLS LAST.
+   *   // Scala: sort a DataFrame by age column in ascending order and null values appearing first.
    *   df.sort(df("age").asc_nulls_last)
    *
    *   // Java
@@ -1062,8 +1077,22 @@ class Column(protected[sql] val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 2.1.0
    */
-  def asc_nulls_last: Column = withExpr { SortOrder(expr, Ascending, NullsLast) }
+  def asc_nulls_first: Column = withExpr { SortOrder(expr, Ascending, NullsFirst) }
 
+  /**
+   * Returns an ordering used in sorting, where null values appear after non-null values.
+   * {{{
+   *   // Scala: sort a DataFrame by age column in ascending order and null values appearing last.
+   *   df.sort(df("age").asc_nulls_last)
+   *
+   *   // Java
+   *   df.sort(df.col("age").asc_nulls_last());
+   * }}}
+   *
+   * @group expr_ops
+   * @since 2.1.0
+   */
+  def asc_nulls_last: Column = withExpr { SortOrder(expr, Ascending, NullsLast) }
 
   /**
    * Prints the expression to the console for debugging purpose.

sql/core/src/main/scala/org/apache/spark/sql/functions.scala

@@ -109,7 +109,6 @@ object functions {
   /**
    * Returns a sort expression based on ascending order of the column.
    * {{{
-   *   // Sort by dept in ascending order, and then age in descending order.
    *   df.sort(asc("dept"), desc("age"))
    * }}}
    *
@@ -119,22 +118,32 @@ object functions {
   def asc(columnName: String): Column = Column(columnName).asc
 
   /**
-   * Returns a sort expression based on ascending order of the column with NULLS LAST.
+   * Returns a sort expression based on ascending order of the column,
+   * and null values return before non-null values.
    * {{{
-   *   // Sort by dept in ascending order nulls first, and then age in descending order.
    *   df.sort(asc_nulls_last("dept"), desc("age"))
    * }}}
    *
    * @group sort_funcs
    * @since 2.1.0
    */
-  def asc_nulls_last(columnName: String): Column = Column(columnName).asc_nulls_last
+  def asc_nulls_first(columnName: String): Column = Column(columnName).asc_nulls_first
 
+  /**
+   * Returns a sort expression based on ascending order of the column,
+   * and null values appear after non-null values.
+   * {{{
+   *   df.sort(asc_nulls_last("dept"), desc("age"))
+   * }}}
+   *
+   * @group sort_funcs
+   * @since 2.1.0
+   */
+  def asc_nulls_last(columnName: String): Column = Column(columnName).asc_nulls_last
 
   /**
    * Returns a sort expression based on the descending order of the column.
    * {{{
-   *   // Sort by dept in ascending order, and then age in descending order.
    *   df.sort(asc("dept"), desc("age"))
    * }}}
    *
@@ -144,9 +153,9 @@ object functions {
   def desc(columnName: String): Column = Column(columnName).desc
 
   /**
-   * Returns a sort expression based on the descending order of the column with NULLS FIRST.
+   * Returns a sort expression based on the descending order of the column,
+   * and null values appear before non-null values.
    * {{{
-   *   // Sort by dept in ascending order, and then age in descending order NULLS FIRST.
    *   df.sort(asc("dept"), desc_nulls_first("age"))
    * }}}
    *
@@ -155,6 +164,18 @@ object functions {
    */
   def desc_nulls_first(columnName: String): Column = Column(columnName).desc_nulls_first
 
+  /**
+   * Returns a sort expression based on the descending order of the column,
+   * and null values appear after non-null values.
+   * {{{
+   *   df.sort(asc("dept"), desc_nulls_last("age"))
+   * }}}
+   *
+   * @group sort_funcs
+   * @since 2.1.0
+   */
+  def desc_nulls_last(columnName: String): Column = Column(columnName).desc_nulls_last
+
 
   //////////////////////////////////////////////////////////////////////////////////////////////
   // Aggregate functions

sql/core/src/test/resources/sql-tests/inputs/orderby-nulls-ordering.sql renamed to sql/core/src/test/resources/sql-tests/inputs/order-by-nulls-ordering.sql

@@ -326,37 +326,21 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
   }
 
   test("sorting with null ordering") {
-    checkAnswer(
-      nullableData.orderBy('a.asc_nulls_last, 'b.desc_nulls_first),
-      Seq(
-        Row(2, null), Row(2, "B"), Row(3, null), Row(4, "a"),
-        Row(5, "A"), Row(null, "c"), Row(null, "b")
-      )
-    )
-
-    checkAnswer(
-      nullableData.orderBy(asc_nulls_last("a"), desc_nulls_first("b")),
-      Seq(
-        Row(2, null), Row(2, "B"), Row(3, null), Row(4, "a"),
-        Row(5, "A"), Row(null, "c"), Row(null, "b")
-      )
-    )
-
-    checkAnswer(
-      nullableData.orderBy('a.desc_nulls_first, 'b.asc_nulls_last),
-      Seq(
-        Row(null, "b"), Row(null, "c"), Row(5, "A"), Row(4, "a"),
-        Row(3, null), Row(2, "B"), Row(2, null)
-      )
-    )
-
-    checkAnswer(
-      nullableData.orderBy(desc_nulls_first("a"), asc_nulls_last("b")),
-      Seq(
-        Row(null, "b"), Row(null, "c"), Row(5, "A"), Row(4, "a"),
-        Row(3, null), Row(2, "B"), Row(2, null)
-      )
-    )
+    val data = Seq[java.lang.Integer](2, 1, null).toDF("key")
+
+    checkAnswer(data.orderBy('key.asc), Row(null) :: Row(1) :: Row(2) :: Nil)
+    checkAnswer(data.orderBy(asc("key")), Row(null) :: Row(1) :: Row(2) :: Nil)
+    checkAnswer(data.orderBy('key.asc_nulls_first), Row(null) :: Row(1) :: Row(2) :: Nil)
+    checkAnswer(data.orderBy(asc_nulls_first("key")), Row(null) :: Row(1) :: Row(2) :: Nil)
+    checkAnswer(data.orderBy('key.asc_nulls_last), Row(1) :: Row(2) :: Row(null) :: Nil)
+    checkAnswer(data.orderBy(asc_nulls_last("key")), Row(1) :: Row(2) :: Row(null) :: Nil)
+
+    checkAnswer(data.orderBy('key.desc), Row(2) :: Row(1) :: Row(null) :: Nil)
+    checkAnswer(data.orderBy(desc("key")), Row(2) :: Row(1) :: Row(null) :: Nil)
+    checkAnswer(data.orderBy('key.desc_nulls_first), Row(null) :: Row(2) :: Row(1) :: Nil)
+    checkAnswer(data.orderBy(desc_nulls_first("key")), Row(null) :: Row(2) :: Row(1) :: Nil)
+    checkAnswer(data.orderBy('key.desc_nulls_last), Row(2) :: Row(1) :: Row(null) :: Nil)
+    checkAnswer(data.orderBy(desc_nulls_last("key")), Row(2) :: Row(1) :: Row(null) :: Nil)
   }
 
   test("global sorting") {

sql/core/src/test/resources/sql-tests/results/orderby-nulls-ordering.sql.out renamed to sql/core/src/test/resources/sql-tests/results/order-by-nulls-ordering.sql.out

@@ -169,20 +169,6 @@ private[sql] trait SQLTestData { self =>
     rdd
   }
 
-  protected lazy val nullableData: DataFrame = {
-    val df = spark.sparkContext.parallelize(
-      NullableRecord(4, "a") ::
-      NullableRecord(null, "c") ::
-      NullableRecord(2, null) ::
-      NullableRecord(null, "b") ::
-      NullableRecord(3, null) ::
-      NullableRecord(5, "A") ::
-      NullableRecord(2, "B") :: Nil, 2
-    ).toDF("a", "b")
-    df.createOrReplaceTempView("nullableData")
-    df
-  }
-
   protected lazy val nullInts: DataFrame = {
     val df = spark.sparkContext.parallelize(
       NullInts(1) ::
@@ -319,7 +305,6 @@ private[sql] object SQLTestData {
   case class IntField(i: Int)
   case class NullInts(a: Integer)
   case class NullStrings(n: Int, s: String)
-  case class NullableRecord(n: Integer, s: String)
   case class TableName(tableName: String)
   case class Person(id: Int, name: String, age: Int)
   case class Salary(personId: Int, salary: Double)