Skip to content

Commit b84ff7e

Browse files
gatorsmilehvanhovell
authored andcommitted
[SPARK-20719][SQL] Support LIMIT ALL
### What changes were proposed in this pull request? `LIMIT ALL` is the same as omitting the `LIMIT` clause. It is supported by both PrestgreSQL and Presto. This PR is to support it by adding it in the parser. ### How was this patch tested? Added a test case Author: Xiao Li <[email protected]> Closes #17960 from gatorsmile/LimitAll.
1 parent e3d2022 commit b84ff7e

File tree

5 files changed

+49
-34
lines changed

5 files changed

+49
-34
lines changed

sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ queryOrganization
334334
(DISTRIBUTE BY distributeBy+=expression (',' distributeBy+=expression)*)?
335335
(SORT BY sort+=sortItem (',' sort+=sortItem)*)?
336336
windows?
337-
(LIMIT limit=expression)?
337+
(LIMIT (ALL | limit=expression))?
338338
;
339339

340340
multiInsertQueryBody

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
281281
val withWindow = withOrder.optionalMap(windows)(withWindows)
282282

283283
// LIMIT
284+
// - LIMIT ALL is the same as omitting the LIMIT clause
284285
withWindow.optional(limit) {
285286
Limit(typedVisit(limit), withWindow)
286287
}
Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,27 @@
11

22
-- limit on various data types
3-
select * from testdata limit 2;
4-
select * from arraydata limit 2;
5-
select * from mapdata limit 2;
3+
SELECT * FROM testdata LIMIT 2;
4+
SELECT * FROM arraydata LIMIT 2;
5+
SELECT * FROM mapdata LIMIT 2;
66

77
-- foldable non-literal in limit
8-
select * from testdata limit 2 + 1;
8+
SELECT * FROM testdata LIMIT 2 + 1;
99

10-
select * from testdata limit CAST(1 AS int);
10+
SELECT * FROM testdata LIMIT CAST(1 AS int);
1111

1212
-- limit must be non-negative
13-
select * from testdata limit -1;
13+
SELECT * FROM testdata LIMIT -1;
14+
SELECT * FROM testData TABLESAMPLE (-1 ROWS);
1415

1516
-- limit must be foldable
16-
select * from testdata limit key > 3;
17+
SELECT * FROM testdata LIMIT key > 3;
1718

1819
-- limit must be integer
19-
select * from testdata limit true;
20-
select * from testdata limit 'a';
20+
SELECT * FROM testdata LIMIT true;
21+
SELECT * FROM testdata LIMIT 'a';
2122

2223
-- limit within a subquery
23-
select * from (select * from range(10) limit 5) where id > 3;
24+
SELECT * FROM (SELECT * FROM range(10) LIMIT 5) WHERE id > 3;
25+
26+
-- limit ALL
27+
SELECT * FROM testdata WHERE key < 3 LIMIT ALL;
Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 10
2+
-- Number of queries: 12
33

44

55
-- !query 0
6-
select * from testdata limit 2
6+
SELECT * FROM testdata LIMIT 2
77
-- !query 0 schema
88
struct<key:int,value:string>
99
-- !query 0 output
@@ -12,7 +12,7 @@ struct<key:int,value:string>
1212

1313

1414
-- !query 1
15-
select * from arraydata limit 2
15+
SELECT * FROM arraydata LIMIT 2
1616
-- !query 1 schema
1717
struct<arraycol:array<int>,nestedarraycol:array<array<int>>>
1818
-- !query 1 output
@@ -21,7 +21,7 @@ struct<arraycol:array<int>,nestedarraycol:array<array<int>>>
2121

2222

2323
-- !query 2
24-
select * from mapdata limit 2
24+
SELECT * FROM mapdata LIMIT 2
2525
-- !query 2 schema
2626
struct<mapcol:map<int,string>>
2727
-- !query 2 output
@@ -30,7 +30,7 @@ struct<mapcol:map<int,string>>
3030

3131

3232
-- !query 3
33-
select * from testdata limit 2 + 1
33+
SELECT * FROM testdata LIMIT 2 + 1
3434
-- !query 3 schema
3535
struct<key:int,value:string>
3636
-- !query 3 output
@@ -40,15 +40,15 @@ struct<key:int,value:string>
4040

4141

4242
-- !query 4
43-
select * from testdata limit CAST(1 AS int)
43+
SELECT * FROM testdata LIMIT CAST(1 AS int)
4444
-- !query 4 schema
4545
struct<key:int,value:string>
4646
-- !query 4 output
4747
1 1
4848

4949

5050
-- !query 5
51-
select * from testdata limit -1
51+
SELECT * FROM testdata LIMIT -1
5252
-- !query 5 schema
5353
struct<>
5454
-- !query 5 output
@@ -57,35 +57,53 @@ The limit expression must be equal to or greater than 0, but got -1;
5757

5858

5959
-- !query 6
60-
select * from testdata limit key > 3
60+
SELECT * FROM testData TABLESAMPLE (-1 ROWS)
6161
-- !query 6 schema
6262
struct<>
6363
-- !query 6 output
6464
org.apache.spark.sql.AnalysisException
65-
The limit expression must evaluate to a constant value, but got (testdata.`key` > 3);
65+
The limit expression must be equal to or greater than 0, but got -1;
6666

6767

6868
-- !query 7
69-
select * from testdata limit true
69+
SELECT * FROM testdata LIMIT key > 3
7070
-- !query 7 schema
7171
struct<>
7272
-- !query 7 output
7373
org.apache.spark.sql.AnalysisException
74-
The limit expression must be integer type, but got boolean;
74+
The limit expression must evaluate to a constant value, but got (testdata.`key` > 3);
7575

7676

7777
-- !query 8
78-
select * from testdata limit 'a'
78+
SELECT * FROM testdata LIMIT true
7979
-- !query 8 schema
8080
struct<>
8181
-- !query 8 output
8282
org.apache.spark.sql.AnalysisException
83-
The limit expression must be integer type, but got string;
83+
The limit expression must be integer type, but got boolean;
8484

8585

8686
-- !query 9
87-
select * from (select * from range(10) limit 5) where id > 3
87+
SELECT * FROM testdata LIMIT 'a'
8888
-- !query 9 schema
89-
struct<id:bigint>
89+
struct<>
9090
-- !query 9 output
91+
org.apache.spark.sql.AnalysisException
92+
The limit expression must be integer type, but got string;
93+
94+
95+
-- !query 10
96+
SELECT * FROM (SELECT * FROM range(10) LIMIT 5) WHERE id > 3
97+
-- !query 10 schema
98+
struct<id:bigint>
99+
-- !query 10 output
91100
4
101+
102+
103+
-- !query 11
104+
SELECT * FROM testdata WHERE key < 3 LIMIT ALL
105+
-- !query 11 schema
106+
struct<key:int,value:string>
107+
-- !query 11 output
108+
1 1
109+
2 2

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -523,14 +523,6 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
523523
sortTest()
524524
}
525525

526-
test("negative in LIMIT or TABLESAMPLE") {
527-
val expected = "The limit expression must be equal to or greater than 0, but got -1"
528-
var e = intercept[AnalysisException] {
529-
sql("SELECT * FROM testData TABLESAMPLE (-1 rows)")
530-
}.getMessage
531-
assert(e.contains(expected))
532-
}
533-
534526
test("CTE feature") {
535527
checkAnswer(
536528
sql("with q1 as (select * from testData limit 10) select * from q1"),

0 commit comments

Comments
 (0)