Skip to content

Commit 0066d6f

Browse files
maropugatorsmile
authored andcommitted
[SPARK-21213][SQL][FOLLOWUP] Use compatible types for comparisons in compareAndGetNewStats
## What changes were proposed in this pull request? This pr fixed code to compare values in `compareAndGetNewStats`. The test below fails in the current master; ``` val oldStats2 = CatalogStatistics(sizeInBytes = BigInt(Long.MaxValue) * 2) val newStats5 = CommandUtils.compareAndGetNewStats( Some(oldStats2), newTotalSize = BigInt(Long.MaxValue) * 2, None) assert(newStats5.isEmpty) ``` ## How was this patch tested? Added some tests in `CommandUtilsSuite`. Author: Takeshi Yamamuro <[email protected]> Closes #20245 from maropu/SPARK-21213-FOLLOWUP.
1 parent ba891ec commit 0066d6f

File tree

2 files changed

+58
-2
lines changed

2 files changed

+58
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,8 @@ object CommandUtils extends Logging {
116116
oldStats: Option[CatalogStatistics],
117117
newTotalSize: BigInt,
118118
newRowCount: Option[BigInt]): Option[CatalogStatistics] = {
119-
val oldTotalSize = oldStats.map(_.sizeInBytes.toLong).getOrElse(-1L)
120-
val oldRowCount = oldStats.flatMap(_.rowCount.map(_.toLong)).getOrElse(-1L)
119+
val oldTotalSize = oldStats.map(_.sizeInBytes).getOrElse(BigInt(-1))
120+
val oldRowCount = oldStats.flatMap(_.rowCount).getOrElse(BigInt(-1))
121121
var newStats: Option[CatalogStatistics] = None
122122
if (newTotalSize >= 0 && newTotalSize != oldTotalSize) {
123123
newStats = Some(CatalogStatistics(sizeInBytes = newTotalSize))
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.execution.command
19+
20+
import org.apache.spark.SparkFunSuite
21+
import org.apache.spark.sql.catalyst.catalog.CatalogStatistics
22+
23+
class CommandUtilsSuite extends SparkFunSuite {
24+
25+
test("Check if compareAndGetNewStats returns correct results") {
26+
val oldStats1 = CatalogStatistics(sizeInBytes = 10, rowCount = Some(100))
27+
val newStats1 = CommandUtils.compareAndGetNewStats(
28+
Some(oldStats1), newTotalSize = 10, newRowCount = Some(100))
29+
assert(newStats1.isEmpty)
30+
val newStats2 = CommandUtils.compareAndGetNewStats(
31+
Some(oldStats1), newTotalSize = -1, newRowCount = None)
32+
assert(newStats2.isEmpty)
33+
val newStats3 = CommandUtils.compareAndGetNewStats(
34+
Some(oldStats1), newTotalSize = 20, newRowCount = Some(-1))
35+
assert(newStats3.isDefined)
36+
newStats3.foreach { stat =>
37+
assert(stat.sizeInBytes === 20)
38+
assert(stat.rowCount.isEmpty)
39+
}
40+
val newStats4 = CommandUtils.compareAndGetNewStats(
41+
Some(oldStats1), newTotalSize = -1, newRowCount = Some(200))
42+
assert(newStats4.isDefined)
43+
newStats4.foreach { stat =>
44+
assert(stat.sizeInBytes === 10)
45+
assert(stat.rowCount.isDefined && stat.rowCount.get === 200)
46+
}
47+
}
48+
49+
test("Check if compareAndGetNewStats can handle large values") {
50+
// Tests for large values
51+
val oldStats2 = CatalogStatistics(sizeInBytes = BigInt(Long.MaxValue) * 2)
52+
val newStats5 = CommandUtils.compareAndGetNewStats(
53+
Some(oldStats2), newTotalSize = BigInt(Long.MaxValue) * 2, None)
54+
assert(newStats5.isEmpty)
55+
}
56+
}

0 commit comments

Comments
 (0)