Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -314,8 +314,10 @@ object FunctionRegistry {
expression[CollectSet]("collect_set"),
expression[CountMinSketchAgg]("count_min_sketch"),
expression[EveryAgg]("every"),
expression[EveryAgg]("bool_and"),
expression[AnyAgg]("any"),
expression[SomeAgg]("some"),
expression[AnyAgg]("some"),
expression[AnyAgg]("bool_or"),

// string functions
expression[Ascii]("ascii"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,19 +71,3 @@ case class EveryAgg(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
case class AnyAgg(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
override def nodeName: String = "Any"
}

@ExpressionDescription(
usage = "_FUNC_(expr) - Returns true if at least one value of `expr` is true.",
examples = """
Examples:
> SELECT _FUNC_(col) FROM VALUES (true), (false), (false) AS tab(col);
true
> SELECT _FUNC_(col) FROM VALUES (NULL), (true), (false) AS tab(col);
true
> SELECT _FUNC_(col) FROM VALUES (false), (false), (NULL) AS tab(col);
false
""",
since = "3.0.0")
case class SomeAgg(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
override def nodeName: String = "Some"
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ object ReplaceExpressions extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
case e: RuntimeReplaceable => e.child
case CountIf(predicate) => Count(new NullIf(predicate, Literal.FalseLiteral))
case SomeAgg(arg) => Max(arg)
case AnyAgg(arg) => Max(arg)
case EveryAgg(arg) => Min(arg)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,6 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
assertSuccess(Min('arrayField))
assertSuccess(new EveryAgg('booleanField))
assertSuccess(new AnyAgg('booleanField))
assertSuccess(new SomeAgg('booleanField))

assertError(Min('mapField), "min does not support ordering on type")
assertError(Max('mapField), "max does not support ordering on type")
Expand Down
18 changes: 13 additions & 5 deletions sql/core/src/test/resources/sql-tests/inputs/group-by.sql
Original file line number Diff line number Diff line change
Expand Up @@ -90,16 +90,16 @@ CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES
(5, null), (5, true), (5, false) AS test_agg(k, v);

-- empty table
SELECT every(v), some(v), any(v) FROM test_agg WHERE 1 = 0;
SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE 1 = 0;

-- all null values
SELECT every(v), some(v), any(v) FROM test_agg WHERE k = 4;
SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE k = 4;

-- aggregates are null Filtering
SELECT every(v), some(v), any(v) FROM test_agg WHERE k = 5;
SELECT every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg WHERE k = 5;

-- group by
SELECT k, every(v), some(v), any(v) FROM test_agg GROUP BY k;
SELECT k, every(v), some(v), any(v), bool_and(v), bool_or(v) FROM test_agg GROUP BY k;

-- having
SELECT k, every(v) FROM test_agg GROUP BY k HAVING every(v) = false;
Expand Down Expand Up @@ -137,10 +137,18 @@ SELECT any(1L);
-- input type checking String
SELECT every("true");

-- every/some/any aggregates are supported as windows expression.
-- input type checking Decimal
SELECT bool_and(1.0);

-- input type checking double
SELECT bool_or(1.0D);

-- every/some/any aggregates/bool_and/bool_or are supported as windows expression.
SELECT k, v, every(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;
SELECT k, v, some(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;
SELECT k, v, any(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;
SELECT k, v, bool_and(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;
SELECT k, v, bool_or(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;

-- Having referencing aggregate expressions is ok.
SELECT count(*) FROM test_agg HAVING count(*) > 1L;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,50 +114,40 @@ SELECT
NOT (FALSE OR FALSE) AS `t`;

-- [SPARK-27880] Implement boolean aggregates(BOOL_AND, BOOL_OR and EVERY)
-- CREATE TEMPORARY TABLE bool_test(
-- b1 BOOL,
-- b2 BOOL,
-- b3 BOOL,
-- b4 BOOL);
CREATE OR REPLACE TEMPORARY VIEW bool_test AS SELECT * FROM VALUES
(TRUE, null, FALSE, null),
(FALSE, TRUE, null, null),
(null, TRUE, FALSE, null) AS bool_test(b1, b2, b3, b4);

-- empty case
-- SELECT
-- BOOL_AND(b1) AS "n",
-- BOOL_OR(b3) AS "n"
-- FROM bool_test;

-- COPY bool_test FROM STDIN NULL 'null';
-- TRUE null FALSE null
-- FALSE TRUE null null
-- null TRUE FALSE null
-- \.
SELECT BOOL_AND(b1) AS n1, BOOL_OR(b3) AS n2 FROM bool_test WHERE 1 = 0;

-- SELECT
-- BOOL_AND(b1) AS "f",
-- BOOL_AND(b2) AS "t",
-- BOOL_AND(b3) AS "f",
-- BOOL_AND(b4) AS "n",
-- BOOL_AND(NOT b2) AS "f",
-- BOOL_AND(NOT b3) AS "t"
-- FROM bool_test;
SELECT
BOOL_AND(b1) AS f1,
BOOL_AND(b2) AS t2,
BOOL_AND(b3) AS f3,
BOOL_AND(b4) AS n4,
BOOL_AND(NOT b2) AS f5,
BOOL_AND(NOT b3) AS t6
FROM bool_test;

-- SELECT
-- EVERY(b1) AS "f",
-- EVERY(b2) AS "t",
-- EVERY(b3) AS "f",
-- EVERY(b4) AS "n",
-- EVERY(NOT b2) AS "f",
-- EVERY(NOT b3) AS "t"
-- FROM bool_test;
SELECT
EVERY(b1) AS f1,
EVERY(b2) AS t2,
EVERY(b3) AS f3,
EVERY(b4) AS n4,
EVERY(NOT b2) AS f5,
EVERY(NOT b3) AS t6
FROM bool_test;

-- SELECT
-- BOOL_OR(b1) AS "t",
-- BOOL_OR(b2) AS "t",
-- BOOL_OR(b3) AS "f",
-- BOOL_OR(b4) AS "n",
-- BOOL_OR(NOT b2) AS "f",
-- BOOL_OR(NOT b3) AS "t"
-- FROM bool_test;
SELECT
BOOL_OR(b1) AS t1,
BOOL_OR(b2) AS t2,
BOOL_OR(b3) AS f3,
BOOL_OR(b4) AS n4,
BOOL_OR(NOT b2) AS f5,
BOOL_OR(NOT b3) AS t6
FROM bool_test;

--
-- Test cases that should be optimized into indexscans instead of
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,50 +116,40 @@ SELECT
NOT (FALSE OR FALSE) AS `t`;

-- [SPARK-27880] Implement boolean aggregates(BOOL_AND, BOOL_OR and EVERY)
-- CREATE TEMPORARY TABLE bool_test(
-- b1 BOOL,
-- b2 BOOL,
-- b3 BOOL,
-- b4 BOOL);
CREATE OR REPLACE TEMPORARY VIEW bool_test AS SELECT * FROM VALUES
(TRUE, null, FALSE, null),
(FALSE, TRUE, null, null),
(null, TRUE, FALSE, null) AS bool_test(b1, b2, b3, b4);

-- empty case
-- SELECT
-- BOOL_AND(b1) AS "n",
-- BOOL_OR(b3) AS "n"
-- FROM bool_test;

-- COPY bool_test FROM STDIN NULL 'null';
-- TRUE null FALSE null
-- FALSE TRUE null null
-- null TRUE FALSE null
-- \.
SELECT BOOL_AND(b1) AS n1, BOOL_OR(b3) AS n2 FROM bool_test WHERE 1 = 0;

-- SELECT
-- BOOL_AND(b1) AS "f",
-- BOOL_AND(b2) AS "t",
-- BOOL_AND(b3) AS "f",
-- BOOL_AND(b4) AS "n",
-- BOOL_AND(NOT b2) AS "f",
-- BOOL_AND(NOT b3) AS "t"
-- FROM bool_test;
SELECT
BOOL_AND(b1) AS f1,
BOOL_AND(b2) AS t2,
BOOL_AND(b3) AS f3,
BOOL_AND(b4) AS n4,
BOOL_AND(NOT b2) AS f5,
BOOL_AND(NOT b3) AS t6
FROM bool_test;

-- SELECT
-- EVERY(b1) AS "f",
-- EVERY(b2) AS "t",
-- EVERY(b3) AS "f",
-- EVERY(b4) AS "n",
-- EVERY(NOT b2) AS "f",
-- EVERY(NOT b3) AS "t"
-- FROM bool_test;
SELECT
EVERY(b1) AS f1,
EVERY(b2) AS t2,
EVERY(b3) AS f3,
EVERY(b4) AS n4,
EVERY(NOT b2) AS f5,
EVERY(NOT b3) AS t6
FROM bool_test;

-- SELECT
-- BOOL_OR(b1) AS "t",
-- BOOL_OR(b2) AS "t",
-- BOOL_OR(b3) AS "f",
-- BOOL_OR(b4) AS "n",
-- BOOL_OR(NOT b2) AS "f",
-- BOOL_OR(NOT b3) AS "t"
-- FROM bool_test;
SELECT
BOOL_OR(b1) AS t1,
BOOL_OR(b2) AS t2,
BOOL_OR(b3) AS f3,
BOOL_OR(b4) AS n4,
BOOL_OR(NOT b2) AS f5,
BOOL_OR(NOT b3) AS t6
FROM bool_test;

--
-- Test cases that should be optimized into indexscans instead of
Expand Down
Loading