Skip to content

Commit 2048c97

Browse files
committed
address comments.
1 parent 1e4182d commit 2048c97

File tree

7 files changed

+6
-22
lines changed

7 files changed

+6
-22
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -436,10 +436,7 @@ private[hive] class HiveClientImpl(
436436

437437
val totalSize = properties.get(StatsSetupConst.TOTAL_SIZE).map(BigInt(_))
438438
val rawDataSize = properties.get(StatsSetupConst.RAW_DATA_SIZE).map(BigInt(_))
439-
lazy val rowCount = properties.get(StatsSetupConst.ROW_COUNT).map(BigInt(_)) match {
440-
case Some(c) if c >= 0 => Some(c)
441-
case _ => None
442-
}
439+
val rowCount = properties.get(StatsSetupConst.ROW_COUNT).map(BigInt(_)).filter(_ >= 0)
443440
// TODO: check if this estimate is valid for tables after partition pruning.
444441
// NOTE: getting `totalSize` directly from params is kind of hacky, but this should be
445442
// relatively cheap if parameters for the table are populated into the metastore.

sql/hive/src/test/java/org/apache/spark/sql/hive/test/TestHiveSingleton.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@ package org.apache.spark.sql.hive.test
1919

2020
import org.scalatest.BeforeAndAfterAll
2121

22-
import org.apache.spark.sql.SparkSession
2322
import org.apache.spark.SparkFunSuite
23+
import org.apache.spark.sql.SparkSession
24+
import org.apache.spark.sql.hive.HiveExternalCatalog
25+
import org.apache.spark.sql.hive.client.HiveClient
2426

2527

2628
trait TestHiveSingleton extends SparkFunSuite with BeforeAndAfterAll {
2729
protected val spark: SparkSession = TestHive.sparkSession
2830
protected val hiveContext: TestHiveContext = TestHive
31+
protected val hiveClient: HiveClient =
32+
spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
2933

3034
protected override def afterAll(): Unit = {
3135
try {

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -646,7 +646,6 @@ class HiveDDLCommandSuite extends PlanTest with SQLTestUtils with TestHiveSingle
646646
}
647647

648648
test("SPARK-15887: hive-site.xml should be loaded") {
649-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
650649
assert(hiveClient.getConf("hive.in.test", "") == "true")
651650
}
652651

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogBackwardCompatibilitySuite.scala

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,6 @@ import org.apache.spark.util.Utils
3535
class HiveExternalCatalogBackwardCompatibilitySuite extends QueryTest
3636
with SQLTestUtils with TestHiveSingleton with BeforeAndAfterEach {
3737

38-
// To test `HiveExternalCatalog`, we need to read/write the raw table meta from/to hive client.
39-
val hiveClient: HiveClient =
40-
spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
41-
4238
val tempDir = Utils.createTempDir().getCanonicalFile
4339
val tempDirUri = tempDir.toURI
4440
val tempDirStr = tempDir.getAbsolutePath

sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,6 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
5252
jsonFilePath = Utils.getSparkClassLoader.getResource("sample.json").getFile
5353
}
5454

55-
// To test `HiveExternalCatalog`, we need to read the raw table metadata(schema, partition
56-
// columns and bucket specification are still in table properties) from hive client.
57-
private def hiveClient: HiveClient =
58-
sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
59-
6055
test("persistent JSON table") {
6156
withTable("jsonTable") {
6257
sql(

sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,6 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
222222
tabName: String,
223223
analyzedBySpark: Boolean = true,
224224
analyzedByHive: Boolean = true): Unit = {
225-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
226225
sql(
227226
s"""
228227
|CREATE TABLE $tabName (key STRING, value STRING)
@@ -277,7 +276,6 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
277276

278277
// ALTER TABLE SET TBLPROPERTIES invalidates some contents of Hive specific statistics
279278
// This is triggered by the Hive alterTable API
280-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
281279
val describeResult = hiveClient.runSqlHive(s"DESCRIBE FORMATTED $tabName")
282280

283281
val rawDataSize = extractStatsPropValues(describeResult, "rawDataSize")
@@ -303,7 +301,6 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
303301
assert(fetchedStats1 == fetchedStats2)
304302

305303
// ALTER TABLE RENAME does not affect the contents of Hive specific statistics
306-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
307304
val describeResult = hiveClient.runSqlHive(s"DESCRIBE FORMATTED $newName")
308305

309306
val rawDataSize = extractStatsPropValues(describeResult, "rawDataSize")
@@ -328,7 +325,6 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
328325
tabName, hasSizeInBytes = true, expectedRowCounts = Some(500))
329326
assert(fetchedStats1 == fetchedStats2)
330327

331-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
332328
val describeResult = hiveClient.runSqlHive(s"DESCRIBE FORMATTED $tabName")
333329

334330
val totalSize = extractStatsPropValues(describeResult, "totalSize")
@@ -356,7 +352,6 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
356352
tabName, hasSizeInBytes = true, expectedRowCounts = Some(500))
357353
assert(fetchedStats1 == fetchedStats2)
358354

359-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
360355
val describeResult = hiveClient.runSqlHive(s"DESCRIBE FORMATTED $tabName")
361356

362357
val totalSize = extractStatsPropValues(describeResult, "totalSize")
@@ -467,7 +462,6 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
467462
sql(s"analyze table $tableName compute STATISTICS FOR COLUMNS " + stats.keys.mkString(", "))
468463

469464
// Validate statistics
470-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
471465
val table = hiveClient.getTable("default", tableName)
472466

473467
val props = table.properties.filterKeys(_.startsWith("spark.sql.statistics.colStats"))

sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/OrcSourceSuite.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,6 @@ abstract class OrcSuite extends QueryTest with TestHiveSingleton with BeforeAndA
153153
}
154154

155155
test("SPARK-19459/SPARK-18220: read char/varchar column written by Hive") {
156-
val hiveClient = spark.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog].client
157156
val location = Utils.createTempDir()
158157
val uri = location.toURI
159158
try {

0 commit comments

Comments
 (0)