[SPARK-27938][SQL] Remove feature flag LEGACY_PASS_PARTITION_BY_AS_OPTIONS

## What changes were proposed in this pull request?
In PR #24365, we pass in the partitionBy columns as options in `DataFrameWriter`.  To make this change less intrusive for a patch release, we added a feature flag `LEGACY_PASS_PARTITION_BY_AS_OPTIONS` with the default to be false.

For 3.0, we should just do the correct behavior for DSV1, i.e., always passing partitionBy as options, and remove this legacy feature flag.

## How was this patch tested?
Existing tests.

Closes #24784 from liwensun/SPARK-27453-default.

Authored-by: liwensun <[email protected]>
Signed-off-by: HyukjinKwon <[email protected]>

Author: liwensun

sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala

@@ -1711,15 +1711,6 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_PASS_PARTITION_BY_AS_OPTIONS =
-    buildConf("spark.sql.legacy.sources.write.passPartitionByAsOptions")
-      .internal()
-      .doc("Whether to pass the partitionBy columns as options in DataFrameWriter. " +
-        "Data source V1 now silently drops partitionBy columns for non-file-format sources; " +
-        "turning the flag on provides a way for these sources to see these partitionBy columns.")
-      .booleanConf
-      .createWithDefault(false)
-
   val NAME_NON_STRUCT_GROUPING_KEY_AS_VALUE =
     buildConf("spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue")
       .internal()

sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala

@@ -316,12 +316,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   }
 
   private def saveToV1Source(): Unit = {
-    if (SparkSession.active.sessionState.conf.getConf(
-      SQLConf.LEGACY_PASS_PARTITION_BY_AS_OPTIONS)) {
-      partitioningColumns.foreach { columns =>
-        extraOptions += (DataSourceUtils.PARTITIONING_COLUMNS_KEY ->
-          DataSourceUtils.encodePartitioningColumns(columns))
-      }
+    partitioningColumns.foreach { columns =>
+      extraOptions += (DataSourceUtils.PARTITIONING_COLUMNS_KEY ->
+        DataSourceUtils.encodePartitioningColumns(columns))
     }
 
     // Code path for data source v1.

sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala

@@ -225,21 +225,13 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSQLContext with Be
   }
 
   test("pass partitionBy as options") {
-    Seq(true, false).foreach { flag =>
-      withSQLConf(SQLConf.LEGACY_PASS_PARTITION_BY_AS_OPTIONS.key -> s"$flag") {
-        Seq(1).toDF.write
-          .format("org.apache.spark.sql.test")
-          .partitionBy("col1", "col2")
-          .save()
-
-        if (flag) {
-          val partColumns = LastOptions.parameters(DataSourceUtils.PARTITIONING_COLUMNS_KEY)
-          assert(DataSourceUtils.decodePartitioningColumns(partColumns) === Seq("col1", "col2"))
-        } else {
-          assert(!LastOptions.parameters.contains(DataSourceUtils.PARTITIONING_COLUMNS_KEY))
-        }
-      }
-    }
+    Seq(1).toDF.write
+      .format("org.apache.spark.sql.test")
+      .partitionBy("col1", "col2")
+      .save()
+
+    val partColumns = LastOptions.parameters(DataSourceUtils.PARTITIONING_COLUMNS_KEY)
+    assert(DataSourceUtils.decodePartitioningColumns(partColumns) === Seq("col1", "col2"))
   }
 
   test("save mode") {