From e7104b666b2c5c9aef26c44692641cd714901cea Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 25 Mar 2015 14:20:14 -0700 Subject: [PATCH 1/3] remove ::: in SparkR --- R/pkg/R/deserialize.R | 2 +- R/pkg/R/serialize.R | 2 +- R/pkg/R/utils.R | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/R/pkg/R/deserialize.R b/R/pkg/R/deserialize.R index d04ec47e60200..dad6f546f2b91 100644 --- a/R/pkg/R/deserialize.R +++ b/R/pkg/R/deserialize.R @@ -154,7 +154,7 @@ readRowList <- function(obj) { # deserialize the row. rawObj <- rawConnection(obj, "r+") on.exit(close(rawObj)) - SparkR:::readRow(rawObj) + readRow(rawObj) } readRow <- function(inputCon) { diff --git a/R/pkg/R/serialize.R b/R/pkg/R/serialize.R index 08e1215e1bfbe..e78908bfa32fd 100644 --- a/R/pkg/R/serialize.R +++ b/R/pkg/R/serialize.R @@ -101,7 +101,7 @@ writeRowSerialize <- function(outputCon, rows) { serializeRow <- function(row) { rawObj <- rawConnection(raw(0), "wb") on.exit(close(rawObj)) - SparkR:::writeRow(rawObj, row) + writeRow(rawObj, row) rawConnectionValue(rawObj) } diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R index 7dafb33704b52..c337fb0751e72 100644 --- a/R/pkg/R/utils.R +++ b/R/pkg/R/utils.R @@ -299,17 +299,17 @@ getStorageLevel <- function(newLevel = c("DISK_ONLY", "OFF_HEAP")) { match.arg(newLevel) storageLevel <- switch(newLevel, - "DISK_ONLY" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "DISK_ONLY"), - "DISK_ONLY_2" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "DISK_ONLY_2"), - "MEMORY_AND_DISK" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK"), - "MEMORY_AND_DISK_2" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK_2"), - "MEMORY_AND_DISK_SER" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK_SER"), - "MEMORY_AND_DISK_SER_2" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK_SER_2"), - "MEMORY_ONLY" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY"), - "MEMORY_ONLY_2" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY_2"), - "MEMORY_ONLY_SER" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY_SER"), - "MEMORY_ONLY_SER_2" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY_SER_2"), - "OFF_HEAP" = SparkR:::callJStatic("org.apache.spark.storage.StorageLevel", "OFF_HEAP")) + "DISK_ONLY" = callJStatic("org.apache.spark.storage.StorageLevel", "DISK_ONLY"), + "DISK_ONLY_2" = callJStatic("org.apache.spark.storage.StorageLevel", "DISK_ONLY_2"), + "MEMORY_AND_DISK" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK"), + "MEMORY_AND_DISK_2" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK_2"), + "MEMORY_AND_DISK_SER" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK_SER"), + "MEMORY_AND_DISK_SER_2" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_AND_DISK_SER_2"), + "MEMORY_ONLY" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY"), + "MEMORY_ONLY_2" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY_2"), + "MEMORY_ONLY_SER" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY_SER"), + "MEMORY_ONLY_SER_2" = callJStatic("org.apache.spark.storage.StorageLevel", "MEMORY_ONLY_SER_2"), + "OFF_HEAP" = callJStatic("org.apache.spark.storage.StorageLevel", "OFF_HEAP")) } # Utility function for functions where an argument needs to be integer but we want to allow From f8fa8af57f663d9c79188f81fd2f67d5c8af8634 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 25 Mar 2015 14:27:38 -0700 Subject: [PATCH 2/3] mute logging when start/stop context --- R/pkg/R/pairRDD.R | 4 ++-- R/pkg/R/sparkR.R | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/pkg/R/pairRDD.R b/R/pkg/R/pairRDD.R index 7929a29c00e1d..c2396c32a7548 100644 --- a/R/pkg/R/pairRDD.R +++ b/R/pkg/R/pairRDD.R @@ -708,9 +708,9 @@ setMethod("cogroup", acc <- res[[i]] # Create an accumulator. if (is.null(acc)) { - acc <- SparkR:::initAccumulator() + acc <- initAccumulator() } - SparkR:::addItemToAccumulator(acc, x[[2]]) + addItemToAccumulator(acc, x[[2]]) res[[i]] <- acc } lapply(res, function(acc) { diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index b149122dc7247..54887d418012b 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -37,7 +37,7 @@ connExists <- function(env) { sparkR.stop <- function() { env <- .sparkREnv if (exists(".sparkRCon", envir = env)) { - cat("Stopping SparkR\n") + # cat("Stopping SparkR\n") if (exists(".sparkRjsc", envir = env)) { sc <- get(".sparkRjsc", envir = env) callJMethod(sc, "stop") @@ -103,7 +103,7 @@ sparkR.init <- function( sparkRLibDir = "") { if (exists(".sparkRjsc", envir = .sparkREnv)) { - cat("Re-using existing Spark Context. Please stop SparkR with sparkR.stop() or restart R to create a new Spark Context\n") + # cat("Re-using existing Spark Context. Please stop SparkR with sparkR.stop() or restart R to create a new Spark Context\n") return(get(".sparkRjsc", envir = .sparkREnv)) } From a18ff5c84e46910f609220eb85bedde1e85e6ba4 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Mon, 6 Apr 2015 14:28:51 -0700 Subject: [PATCH 3/3] Update sparkR.R --- R/pkg/R/sparkR.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R index 54887d418012b..bc82df01f0fff 100644 --- a/R/pkg/R/sparkR.R +++ b/R/pkg/R/sparkR.R @@ -103,7 +103,7 @@ sparkR.init <- function( sparkRLibDir = "") { if (exists(".sparkRjsc", envir = .sparkREnv)) { - # cat("Re-using existing Spark Context. Please stop SparkR with sparkR.stop() or restart R to create a new Spark Context\n") + cat("Re-using existing Spark Context. Please stop SparkR with sparkR.stop() or restart R to create a new Spark Context\n") return(get(".sparkRjsc", envir = .sparkREnv)) }