Skip to content

[SPARK-26990][SQL][BACKPORT-2.4] FileIndex: use user specified field names if possible #23909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SPARK-26990][SQL][BACKPORT-2.4] FileIndex: use user specified field names if possible #23909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,13 @@ object PartitioningUtils {
Map.empty[String, DataType]
}

// SPARK-26990: use user specified field names if case insensitive.
val userSpecifiedNames = if (userSpecifiedSchema.isDefined && !caseSensitive) {
CaseInsensitiveMap(userSpecifiedSchema.get.fields.map(f => f.name -> f.name).toMap)
} else {
Map.empty[String, String]
}

// First, we need to parse every partition's path and see if we can find partition values.
val (partitionValues, optDiscoveredBasePaths) = paths.map { path =>
parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes, timeZone)
Expand Down Expand Up @@ -163,7 +170,9 @@ object PartitioningUtils {
columnNames.zip(literals).map { case (name, Literal(_, dataType)) =>
// We always assume partition columns are nullable since we've no idea whether null values
// will be appended in the future.
StructField(name, userSpecifiedDataTypes.getOrElse(name, dataType), nullable = true)
val resultName = userSpecifiedNames.getOrElse(name, name)
val resultDataType = userSpecifiedDataTypes.getOrElse(name, dataType)
StructField(resultName, resultDataType, nullable = true)
}
}

Expand Down
15 changes: 15 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,21 @@ class FileIndexSuite extends SharedSQLContext {
}
}

test("SPARK-26990: use user specified field names if possible") {
withTempDir { dir =>
val partitionDirectory = new File(dir, "a=foo")
partitionDirectory.mkdir()
val file = new File(partitionDirectory, "text.txt")
stringToFile(file, "text")
val path = new Path(dir.getCanonicalPath)
val schema = StructType(Seq(StructField("A", StringType, false)))
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
val fileIndex = new InMemoryFileIndex(spark, Seq(path), Map.empty, Some(schema))
assert(fileIndex.partitionSchema.length == 1 && fileIndex.partitionSchema.head.name == "A")
}
}
}

test("InMemoryFileIndex: input paths are converted to qualified paths") {
withTempDir { dir =>
val file = new File(dir, "text.txt")
Expand Down