Skip to content

SPARK-1630: Make PythonRDD handle NULL elements and strings gracefully #554

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

SPARK-1630: Make PythonRDD handle NULL elements and strings gracefully #554

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff036d3
SPARK-1630: Make PythonRDD handle Null elements and strings gracefully
kalpit Apr 25, 2014
8a4a0f9
SPARK-1630: Incorporated code-review feedback
kalpit Apr 25, 2014
dddda9e
SPARK-1630: Fixed indentation
kalpit Apr 25, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ private object SpecialLengths {
val TIMING_DATA = -3
}

private[spark] object PythonRDD {
private[spark] object PythonRDD extends Logging {
val UTF8 = Charset.forName("UTF-8")

def readRDDFromFile(sc: JavaSparkContext, filename: String, parallelism: Int):
Expand Down Expand Up @@ -301,15 +301,23 @@ private[spark] object PythonRDD {
throw new SparkException("Unexpected Tuple2 element type " + pair._1.getClass)
}
case other =>
throw new SparkException("Unexpected element type " + first.getClass)
if (other == null) {
logDebug("Encountered NULL element from iterator. We skip writing NULL to stream.")
} else {
throw new SparkException("Unexpected element type " + first.getClass)
}
}
}
}

def writeUTF(str: String, dataOut: DataOutputStream) {
val bytes = str.getBytes(UTF8)
dataOut.writeInt(bytes.length)
dataOut.write(bytes)
if (str == null) {
logDebug("Encountered NULL string. We skip writing NULL to stream.")
} else {
val bytes = str.getBytes(UTF8)
Copy link
Contributor

@rxin rxin Apr 25, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the indent is off here (2-space indent)

Copy link
Contributor

@rxin rxin Apr 25, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you update this one also?

dataOut.writeInt(bytes.length)
dataOut.write(bytes)
}
}

def writeToFile[T](items: java.util.Iterator[T], filename: String) {
Expand Down
15 changes: 10 additions & 5 deletions core/src/test/scala/org/apache/spark/api/python/PythonRDDSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,16 @@ import org.scalatest.FunSuite

class PythonRDDSuite extends FunSuite {

test("Writing large strings to the worker") {
val input: List[String] = List("a"*100000)
val buffer = new DataOutputStream(new ByteArrayOutputStream)
PythonRDD.writeIteratorToStream(input.iterator, buffer)
}
test("Writing large strings to the worker") {
val input: List[String] = List("a"*100000)
val buffer = new DataOutputStream(new ByteArrayOutputStream)
PythonRDD.writeIteratorToStream(input.iterator, buffer)
}

test("Handle nulls gracefully") {
val input: List[String] = List("a", null)
val buffer = new DataOutputStream(new ByteArrayOutputStream)
PythonRDD.writeIteratorToStream(input.iterator, buffer)
}
}