Skip to content

Commit 2e07ed3

Browse files
HyukjinKwondongjoon-hyun
authored andcommitted
[SPARK-33082][SPARK-20202][BUILD][SQL][FOLLOW-UP] Remove Hive 1.2 workarounds and Hive 1.2 profile in Jenkins script
### What changes were proposed in this pull request? This PR removes the leftover of Hive 1.2 workarounds and Hive 1.2 profile in Jenkins script. - `test-hive1.2` title is not used anymore in Jenkins - Remove some comments related to Hive 1.2 - Remove unused codes in `OrcFilters.scala` Hive - Test `spark.sql.hive.convertMetastoreOrc` disabled case for the tests added at SPARK-19809 and SPARK-22267 ### Why are the changes needed? To remove unused codes & improve test coverage ### Does this PR introduce _any_ user-facing change? No, dev-only. ### How was this patch tested? Manually ran the unit tests. Also It will be tested in CI in this PR. Closes #29973 from HyukjinKwon/SPARK-33082-SPARK-20202. Authored-by: HyukjinKwon <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent edb140e commit 2e07ed3

File tree

9 files changed

+13
-970
lines changed

9 files changed

+13
-970
lines changed

dev/deps/spark-deps-hadoop-2.7-hive-1.2

Lines changed: 0 additions & 230 deletions
This file was deleted.

dev/run-tests-jenkins.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,8 +175,6 @@ def main():
175175
if "test-hadoop3.2" in ghprb_pull_title:
176176
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop3.2"
177177
# Switch the Hive profile based on the PR title:
178-
if "test-hive1.2" in ghprb_pull_title:
179-
os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive1.2"
180178
if "test-hive2.3" in ghprb_pull_title:
181179
os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive2.3"
182180

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,6 @@ import org.apache.spark.sql.types._
3939

4040
/**
4141
* A test suite that tests Apache ORC filter API based filter pushdown optimization.
42-
* OrcFilterSuite and HiveOrcFilterSuite is logically duplicated to provide the same test coverage.
43-
* The difference are the packages containing 'Predicate' and 'SearchArgument' classes.
44-
* - OrcFilterSuite uses 'org.apache.orc.storage.ql.io.sarg' package.
45-
* - HiveOrcFilterSuite uses 'org.apache.hadoop.hive.ql.io.sarg' package.
4642
*/
4743
class OrcFilterSuite extends OrcTest with SharedSparkSession {
4844

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,6 @@ abstract class OrcQueryTest extends OrcTest {
217217
}
218218
}
219219

220-
// Hive supports zlib, snappy and none for Hive 1.2.1.
221220
test("Compression options for writing to an ORC file (SNAPPY, ZLIB and NONE)") {
222221
withTempPath { file =>
223222
spark.range(0, 10).write

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ import org.apache.spark.sql.internal.SQLConf.ORC_IMPLEMENTATION
4646
* -> OrcPartitionDiscoverySuite
4747
* -> HiveOrcPartitionDiscoverySuite
4848
* -> OrcFilterSuite
49-
* -> HiveOrcFilterSuite
5049
*/
5150
abstract class OrcTest extends QueryTest with FileBasedDataSourceTest with BeforeAndAfterAll {
5251

sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ import org.apache.spark.sql.SparkSession
4747
import org.apache.spark.sql.catalyst.InternalRow
4848
import org.apache.spark.sql.catalyst.expressions._
4949
import org.apache.spark.sql.execution.datasources._
50-
import org.apache.spark.sql.execution.datasources.orc.OrcOptions
50+
import org.apache.spark.sql.execution.datasources.orc.{OrcFilters, OrcOptions}
5151
import org.apache.spark.sql.hive.{HiveInspectors, HiveShim}
5252
import org.apache.spark.sql.sources.{Filter, _}
5353
import org.apache.spark.sql.types._
@@ -139,7 +139,7 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable
139139

140140
if (sparkSession.sessionState.conf.orcFilterPushDown) {
141141
// Sets pushed predicates
142-
OrcFilters.createFilter(requiredSchema, filters.toArray).foreach { f =>
142+
OrcFilters.createFilter(requiredSchema, filters).foreach { f =>
143143
hadoopConf.set(OrcFileFormat.SARG_PUSHDOWN, toKryo(f))
144144
hadoopConf.setBoolean(ConfVars.HIVEOPTINDEXFILTER.varname, true)
145145
}
@@ -296,7 +296,7 @@ private[orc] class OrcOutputWriter(
296296

297297
override def close(): Unit = {
298298
if (recordWriterInstantiated) {
299-
// Hive 1.2.1 ORC initializes its private `writer` field at the first write.
299+
// Hive ORC initializes its private `writer` field at the first write.
300300
OrcFileFormat.addSparkVersionMetadata(recordWriter)
301301
recordWriter.close(Reporter.NULL)
302302
}

0 commit comments

Comments
 (0)