Skip to content

[SPARK-19618][SQL] Inconsistency wrt max. buckets allowed from Dataframe API vs SQL #16948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SPARK-19618][SQL] Inconsistency wrt max. buckets allowed from Dataframe API vs SQL #16948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,9 @@ case class BucketSpec(
numBuckets: Int,
bucketColumnNames: Seq[String],
sortColumnNames: Seq[String]) {
if (numBuckets <= 0) {
throw new AnalysisException(s"Expected positive number of buckets, but got `$numBuckets`.")
if (numBuckets <= 0 || numBuckets >= 100000) {
throw new AnalysisException(
s"Number of buckets should be greater than 0 but less than 100000. Got `$numBuckets`")
}

override def toString: String = {
Expand Down
1 change: 0 additions & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,6 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
}

numBuckets.map { n =>
require(n > 0 && n < 100000, "Bucket number must be greater than 0 and less than 100000.")
BucketSpec(n, bucketColumnNames.get, sortColumnNames.getOrElse(Nil))
}
}
Expand Down
28 changes: 15 additions & 13 deletions sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ class CreateTableAsSelectSuite
}
}

test("create table using as select - with non-zero buckets") {
test("create table using as select - with valid number of buckets") {
val catalog = spark.sessionState.catalog
withTable("t") {
sql(
Expand All @@ -222,19 +222,21 @@ class CreateTableAsSelectSuite
}
}

test("create table using as select - with zero buckets") {
test("create table using as select - with invalid number of buckets") {
withTable("t") {
val e = intercept[AnalysisException] {
sql(
s"""
|CREATE TABLE t USING PARQUET
|OPTIONS (PATH '${path.toURI}')
|CLUSTERED BY (a) SORTED BY (b) INTO 0 BUCKETS
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
)
}.getMessage
assert(e.contains("Expected positive number of buckets, but got `0`"))
Seq(0, 100000).foreach(numBuckets => {
val e = intercept[AnalysisException] {
sql(
s"""
|CREATE TABLE t USING PARQUET
|OPTIONS (PATH '${path.toURI}')
|CLUSTERED BY (a) SORTED BY (b) INTO $numBuckets BUCKETS
|AS SELECT 1 AS a, 2 AS b
""".stripMargin
)
}.getMessage
assert(e.contains("Number of buckets should be greater than 0 but less than 100000"))
})
}
}

Expand Down
10 changes: 7 additions & 3 deletions sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,14 @@ class BucketedWriteSuite extends QueryTest with SQLTestUtils with TestHiveSingle
intercept[AnalysisException](df.write.bucketBy(2, "k").saveAsTable("tt"))
}

test("numBuckets not greater than 0 or less than 100000") {
test("numBuckets be greater than 0 but less than 100000") {
val df = Seq(1 -> "a", 2 -> "b").toDF("i", "j")
intercept[IllegalArgumentException](df.write.bucketBy(0, "i").saveAsTable("tt"))
intercept[IllegalArgumentException](df.write.bucketBy(100000, "i").saveAsTable("tt"))

Seq(-1, 0, 100000).foreach(numBuckets => {
val e = intercept[AnalysisException](df.write.bucketBy(numBuckets, "i").saveAsTable("tt"))
assert(
e.getMessage.contains("Number of buckets should be greater than 0 but less than 100000"))
})
}

test("specify sorting columns without bucketing columns") {
Expand Down