[SPARK-12242][SQL] Add DataFrame.transform method

Author: Reynold Xin <[email protected]>

Closes #10226 from rxin/df-transform.

Author: rxin

sql/core/src/main/scala/org/apache/spark/sql/Column.scala

@@ -84,7 +84,7 @@ class TypedColumn[-T, U](
  *   col("`a.column.with.dots`") // Escape `.` in column names.
  *   $"columnName"               // Scala short hand for a named column.
  *   expr("a + 1")               // A column that is constructed from a parsed SQL Expression.
- *   lit("1")                    // A column that produces a literal (constant) value.
+ *   lit("abc")                  // A column that produces a literal (constant) value.
  * }}}
  *
  * [[Column]] objects can be composed to form complex expressions:

sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala

@@ -1421,6 +1421,19 @@ class DataFrame private[sql](
    */
   def first(): Row = head()
 
+  /**
+   * Concise syntax for chaining custom transformations.
+   * {{{
+   *   def featurize(ds: DataFrame) = ...
+   *
+   *   df
+   *     .transform(featurize)
+   *     .transform(...)
+   * }}}
+   * @since 1.6.0
+   */
+  def transform[U](t: DataFrame => DataFrame): DataFrame = t(this)
+
   /**
    * Returns a new RDD by applying a function to all rows of this DataFrame.
    * @group rdd