From a1d66c59abc59a1454a53255ce9e9df66dd98f5a Mon Sep 17 00:00:00 2001 From: tedyu Date: Fri, 12 Jun 2015 15:57:42 -0700 Subject: [PATCH 1/3] Fix NullPointerException with functions.rand() --- .../org/apache/spark/sql/catalyst/expressions/random.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala index 6e4e9cb1be090..fb586afc5ea1e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/random.scala @@ -37,7 +37,11 @@ abstract class RDG(seed: Long) extends LeafExpression with Serializable { * Record ID within each partition. By being transient, the Random Number Generator is * reset every time we serialize and deserialize it. */ - @transient protected lazy val rng = new XORShiftRandom(seed + TaskContext.get().partitionId()) + @transient protected lazy val partitionId = TaskContext.get() match { + case null => 0 + case _ => TaskContext.get().partitionId() + } + @transient protected lazy val rng = new XORShiftRandom(seed + partitionId) override def deterministic: Boolean = false From 750f92cba5afec011faeb0a9e68fa340e8a8e65b Mon Sep 17 00:00:00 2001 From: tedyu Date: Sat, 13 Jun 2015 14:02:30 -0700 Subject: [PATCH 2/3] Add test for Rand() with seed --- .../sql/catalyst/expressions/ArithmeticExpressionSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala index e1afa81a7a82f..34ec545ffd0a8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala @@ -69,6 +69,7 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(-c1, -1.1, row) checkEvaluation(c1 + c2, 3.1, row) + checkDoubleEvaluation(Rand(30), (0.7363714192755834 +- 0.001), row) checkDoubleEvaluation(c1 - c2, (-0.9 +- 0.001), row) checkDoubleEvaluation(c1 * c2, (2.2 +- 0.001), row) checkDoubleEvaluation(c1 / c2, (0.55 +- 0.001), row) From 62fd97b9beb11362f892ff6217848f9fccd72459 Mon Sep 17 00:00:00 2001 From: tedyu Date: Mon, 15 Jun 2015 14:55:15 -0700 Subject: [PATCH 3/3] Create RandomSuite --- .../ArithmeticExpressionSuite.scala | 1 - .../catalyst/expressions/RandomSuite.scala | 33 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala index 34ec545ffd0a8..e1afa81a7a82f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala @@ -69,7 +69,6 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(-c1, -1.1, row) checkEvaluation(c1 + c2, 3.1, row) - checkDoubleEvaluation(Rand(30), (0.7363714192755834 +- 0.001), row) checkDoubleEvaluation(c1 - c2, (-0.9 +- 0.001), row) checkDoubleEvaluation(c1 * c2, (2.2 +- 0.001), row) checkDoubleEvaluation(c1 / c2, (0.55 +- 0.001), row) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala new file mode 100644 index 0000000000000..9be2b23a53f27 --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import org.scalatest.Matchers._ + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.types.{DoubleType, IntegerType} + + +class RandomSuite extends SparkFunSuite with ExpressionEvalHelper { + + test("random") { + val row = create_row(1.1, 2.0, 3.1, null) + checkDoubleEvaluation(Rand(30), (0.7363714192755834 +- 0.001), row) + } +}