Refactor spark.addFile, spark.getSparkFiles API.

Author: yanboliang

R/pkg/NAMESPACE

@@ -336,6 +336,9 @@ export("as.DataFrame",
        "read.parquet",
        "read.text",
        "spark.lapply",
+       "spark.addFile",
+       "spark.getSparkFilesRootDirectory",
+       "spark.getSparkFiles",
        "sql",
        "str",
        "tableToDF",

R/pkg/R/context.R

@@ -229,35 +229,44 @@ setCheckpointDir <- function(sc, dirName) {
 #'
 #' The path passed can be either a local file, a file in HDFS (or other Hadoop-supported
 #' filesystems), or an HTTP, HTTPS or FTP URI. To access the file in Spark jobs,
-#' use sparkFiles.get(fileName) to find its download location.
-#'
-#' A directory can be given if the recursive option is set to true.
-#' Currently directories are only supported for Hadoop-supported filesystems.
+#' use spark.getSparkFiles(fileName) to find its download location.
 #'
+#' @rdname spark.addFile
 #' @param path The path of the file to be added
-#' @param recursive Recursive or not if the path is directory. Default is FALSE.
-#' @noRd
 #' @examples
 #'\dontrun{
-#' sc <- sparkR.init()
-#' addFile(sc, "~/myfile")
+#' spark.addFile("~/myfile")
 #'}
-addFile <- function(sc, path) {
+#' @note spark.addFile since 2.1.0
+spark.addFile <- function(path) {
+  sc <- getSparkContext()
   invisible(callJMethod(sc, "addFile", suppressWarnings(normalizePath(path))))
 }
 
-#' Get the root directory that contains files added through addFile.
+#' Get the root directory that contains files added through spark.addFile.
 #'
-#' @noRd
-sparkFiles.getRootDirectory <- function() {
+#' @rdname spark.getSparkFilesRootDirectory
+#' @return the root directory that contains files added through spark.addFile
+#' @examples
+#'\dontrun{
+#' spark.getSparkFilesRootDirectory()
+#'}
+#' @note spark.getSparkFilesRootDirectory since 2.1.0
+spark.getSparkFilesRootDirectory <- function() {
   callJStatic("org.apache.spark.SparkFiles", "getRootDirectory")
 }
 
-#' Get the absolute path of a file added through addFile.
+#' Get the absolute path of a file added through spark.addFile.
 #'
-#' @param fileName The name of the file added through addFile
-#' @noRd
-sparkFiles.get <- function(fileName) {
+#' @rdname spark.getSparkFiles
+#' @param fileName The name of the file added through spark.addFile
+#' @return the absolute path of a file added through spark.addFile.
+#' @examples
+#'\dontrun{
+#' spark.getSparkFiles("myfile")
+#'}
+#' @note spark.getSparkFiles since 2.1.0
+spark.getSparkFiles <- function(fileName) {
   callJStatic("org.apache.spark.SparkFiles", "get", as.character(fileName))
 }
 

R/pkg/inst/tests/testthat/test_context.R

@@ -166,3 +166,16 @@ test_that("spark.lapply should perform simple transforms", {
   expect_equal(doubled, as.list(2 * 1:10))
   sparkR.session.stop()
 })
+
+test_that("add and get file to be downloaded with Spark job on every node", {
+  sparkR.sparkContext()
+  path <- tempfile(pattern = "hello", fileext = ".txt")
+  filename <- basename(path)
+  words <- "Hello World!"
+  writeLines(words, path)
+  spark.addFile(path)
+  download_path <- spark.getSparkFiles(filename)
+  expect_equal(readLines(download_path), words)
+  unlink(path)
+  sparkR.session.stop()
+})

R/pkg/inst/tests/testthat/test_rdd.R

@@ -801,15 +801,4 @@ test_that("Test correct concurrency of RRDD.compute()", {
   expect_equal(count, 1000)
 })
 
-test_that("add and get file to be downloaded with Spark job on every node", {
-  path <- tempfile(pattern = "hello", fileext = ".txt")
-  filename <- basename(path)
-  words <- "Hello World!"
-  writeLines(words, path)
-  addFile(sc, path)
-  download_path <- sparkFiles.get(filename)
-  expect_equal(readLines(download_path), words)
-  unlink(path)
-})
-
 sparkR.session.stop()