Skip to content

Commit 1ece299

Browse files
LuciferYangdongjoon-hyun
authored andcommitted
[SPARK-45902][SQL] Remove unused function resolvePartitionColumns from DataSource
### What changes were proposed in this pull request? `resolvePartitionColumns` was introduced by SPARK-37287 (#37099) and become unused after SPARK-41713 (#39220), so this pr remove it from `DataSource`. ### Why are the changes needed? Clean up unused code. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GitHub Actions. ### Was this patch authored or co-authored using generative AI tooling? No Closes #43779 from LuciferYang/SPARK-45902. Lead-authored-by: yangjie01 <[email protected]> Co-authored-by: YangJie <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent d2a1a31 commit 1ece299

File tree

1 file changed

+1
-24
lines changed
  • sql/core/src/main/scala/org/apache/spark/sql/execution/datasources

1 file changed

+1
-24
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,8 @@ import org.apache.spark.SparkException
2929
import org.apache.spark.deploy.SparkHadoopUtil
3030
import org.apache.spark.internal.Logging
3131
import org.apache.spark.sql._
32-
import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute}
32+
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
3333
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogUtils}
34-
import org.apache.spark.sql.catalyst.expressions.Attribute
3534
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
3635
import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, TypeUtils}
3736
import org.apache.spark.sql.connector.catalog.TableProvider
@@ -822,26 +821,4 @@ object DataSource extends Logging {
822821
throw QueryCompilationErrors.writeEmptySchemasUnsupportedByDataSourceError()
823822
}
824823
}
825-
826-
/**
827-
* Resolve partition columns using output columns of the query plan.
828-
*/
829-
def resolvePartitionColumns(
830-
partitionColumns: Seq[Attribute],
831-
outputColumns: Seq[Attribute],
832-
plan: LogicalPlan,
833-
resolver: Resolver): Seq[Attribute] = {
834-
partitionColumns.map { col =>
835-
// The partition columns created in `planForWritingFileFormat` should always be
836-
// `UnresolvedAttribute` with a single name part.
837-
assert(col.isInstanceOf[UnresolvedAttribute])
838-
val unresolved = col.asInstanceOf[UnresolvedAttribute]
839-
assert(unresolved.nameParts.length == 1)
840-
val name = unresolved.nameParts.head
841-
outputColumns.find(a => resolver(a.name, name)).getOrElse {
842-
throw QueryCompilationErrors.cannotResolveAttributeError(
843-
name, plan.output.map(_.name).mkString(", "))
844-
}
845-
}
846-
}
847824
}

0 commit comments

Comments
 (0)