From 2b399deeb09af3ff8bb561e84f22948fd589e96d Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Fri, 27 Oct 2017 10:07:46 -0700 Subject: [PATCH 1/2] Set java.io.tmpdir for CRAN checks Also disable the hsperfdata for the JVM for CRAN checks --- R/pkg/inst/tests/testthat/test_basic.R | 12 ++++++++++-- R/pkg/tests/run-all.R | 4 ++++ R/pkg/vignettes/sparkr-vignettes.Rmd | 8 +++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/R/pkg/inst/tests/testthat/test_basic.R b/R/pkg/inst/tests/testthat/test_basic.R index de47162d5325f..55ef8ab8ecc5a 100644 --- a/R/pkg/inst/tests/testthat/test_basic.R +++ b/R/pkg/inst/tests/testthat/test_basic.R @@ -18,7 +18,11 @@ context("basic tests for CRAN") test_that("create DataFrame from list or data.frame", { - sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE) + r_tmp_dir <- tempdir() + tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "") + sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE, + sparkConfig = list(spark.driver.extraJavaOptions = tmp_arg, + spark.executor.extraJavaOptions = tmp_arg)) i <- 4 df <- createDataFrame(data.frame(dummy = 1:i)) @@ -49,7 +53,11 @@ test_that("create DataFrame from list or data.frame", { }) test_that("spark.glm and predict", { - sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE) + r_tmp_dir <- tempdir() + tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "") + sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE, + sparkConfig = list(spark.driver.extraJavaOptions = tmp_arg, + spark.executor.extraJavaOptions = tmp_arg)) training <- suppressWarnings(createDataFrame(iris)) # gaussian family diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R index a1834a220261d..d9018092b738d 100644 --- a/R/pkg/tests/run-all.R +++ b/R/pkg/tests/run-all.R @@ -38,6 +38,10 @@ sparkRFilesBefore <- list.files(path = sparkRDir, all.files = TRUE) sparkRTestMaster <- "local[1]" if (identical(Sys.getenv("NOT_CRAN"), "true")) { sparkRTestMaster <- "" +} else { + # Disable hsperfdata on CRAN + old_java_opt <- Sys.getenv("_JAVA_OPTIONS") + Sys.setenv("_JAVA_OPTIONS"=paste("-XX:-UsePerfData", old_java_opt, sep=" ")) } test_package("SparkR") diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index caeae72e37bbf..367206c8c2b75 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -36,6 +36,12 @@ opts_hooks$set(eval = function(options) { } options }) +r_tmp_dir <- tempdir() +tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "") +sparkSessionConfig <- list(spark.driver.extraJavaOptions = tmp_arg, + spark.executor.extraJavaOptions = tmp_arg) +old_java_opt <- Sys.getenv("_JAVA_OPTIONS") +new_java_opt <- Sys.setenv("_JAVA_OPTIONS"=paste("-XX:-UsePerfData", old_java_opt, sep = " ")) ``` ## Overview @@ -57,7 +63,7 @@ We use default settings in which it runs in local mode. It auto downloads Spark ```{r, include=FALSE} install.spark() -sparkR.session(master = "local[1]") +sparkR.session(master = "local[1]", sparkConfig = sparkSessionConfig, enableHiveSupport = FALSE) ``` ```{r, eval=FALSE} sparkR.session() From 9c8afea8d9e430f223678be0335f474f003565dd Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Sat, 28 Oct 2017 16:05:40 -0700 Subject: [PATCH 2/2] Address comments --- R/pkg/inst/tests/testthat/test_basic.R | 10 ++-------- R/pkg/tests/run-all.R | 7 ++++++- R/pkg/vignettes/sparkr-vignettes.Rmd | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/R/pkg/inst/tests/testthat/test_basic.R b/R/pkg/inst/tests/testthat/test_basic.R index 55ef8ab8ecc5a..823d26f12feee 100644 --- a/R/pkg/inst/tests/testthat/test_basic.R +++ b/R/pkg/inst/tests/testthat/test_basic.R @@ -18,11 +18,8 @@ context("basic tests for CRAN") test_that("create DataFrame from list or data.frame", { - r_tmp_dir <- tempdir() - tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "") sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE, - sparkConfig = list(spark.driver.extraJavaOptions = tmp_arg, - spark.executor.extraJavaOptions = tmp_arg)) + sparkConfig = sparkRTestConfig) i <- 4 df <- createDataFrame(data.frame(dummy = 1:i)) @@ -53,11 +50,8 @@ test_that("create DataFrame from list or data.frame", { }) test_that("spark.glm and predict", { - r_tmp_dir <- tempdir() - tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "") sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE, - sparkConfig = list(spark.driver.extraJavaOptions = tmp_arg, - spark.executor.extraJavaOptions = tmp_arg)) + sparkConfig = sparkRTestConfig) training <- suppressWarnings(createDataFrame(iris)) # gaussian family diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R index d9018092b738d..a7f913e5fad11 100644 --- a/R/pkg/tests/run-all.R +++ b/R/pkg/tests/run-all.R @@ -36,12 +36,17 @@ invisible(lapply(sparkRWhitelistSQLDirs, sparkRFilesBefore <- list.files(path = sparkRDir, all.files = TRUE) sparkRTestMaster <- "local[1]" +sparkRTestConfig <- list() if (identical(Sys.getenv("NOT_CRAN"), "true")) { sparkRTestMaster <- "" } else { # Disable hsperfdata on CRAN old_java_opt <- Sys.getenv("_JAVA_OPTIONS") - Sys.setenv("_JAVA_OPTIONS"=paste("-XX:-UsePerfData", old_java_opt, sep=" ")) + Sys.setenv("_JAVA_OPTIONS" = paste("-XX:-UsePerfData", old_java_opt)) + tmpDir <- tempdir() + tmpArg <- paste0("-Djava.io.tmpdir=", tmpDir) + sparkRTestConfig <- list(spark.driver.extraJavaOptions = tmpArg, + spark.executor.extraJavaOptions = tmpArg) } test_package("SparkR") diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 367206c8c2b75..907bbb3d66018 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -41,7 +41,7 @@ tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "") sparkSessionConfig <- list(spark.driver.extraJavaOptions = tmp_arg, spark.executor.extraJavaOptions = tmp_arg) old_java_opt <- Sys.getenv("_JAVA_OPTIONS") -new_java_opt <- Sys.setenv("_JAVA_OPTIONS"=paste("-XX:-UsePerfData", old_java_opt, sep = " ")) +Sys.setenv("_JAVA_OPTIONS" = paste("-XX:-UsePerfData", old_java_opt, sep = " ")) ``` ## Overview