Skip to content

Commit 359c2e8

Browse files
felixcheungshivaram
felixcheung
authored and
shivaram
committed
[SPARK-15159][SPARKR] SparkSession roxygen2 doc, programming guide, example updates
## What changes were proposed in this pull request? roxygen2 doc, programming guide, example updates ## How was this patch tested? manual checks shivaram Author: Felix Cheung <[email protected]> Closes #13751 from felixcheung/rsparksessiondoc.
1 parent b0f2fb5 commit 359c2e8

File tree

9 files changed

+162
-239
lines changed

9 files changed

+162
-239
lines changed

‎R/pkg/R/DataFrame.R‎

Lines changed: 58 additions & 111 deletions
Large diffs are not rendered by default.

‎R/pkg/R/SQLContext.R‎

Lines changed: 16 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,7 @@ getDefaultSqlSource <- function() {
126126
#' @export
127127
#' @examples
128128
#'\dontrun{
129-
#' sc <- sparkR.init()
130-
#' sqlContext <- sparkRSQL.init(sc)
129+
#' sparkR.session()
131130
#' df1 <- as.DataFrame(iris)
132131
#' df2 <- as.DataFrame(list(3,4,5,6))
133132
#' df3 <- createDataFrame(iris)
@@ -238,8 +237,7 @@ as.DataFrame <- function(x, ...) {
238237
#' @noRd
239238
#' @examples
240239
#'\dontrun{
241-
#' sc <- sparkR.init()
242-
#' sqlContext <- sparkRSQL.init(sc)
240+
#' sparkR.session()
243241
#' rdd <- lapply(parallelize(sc, 1:10), function(x) list(a=x, b=as.character(x)))
244242
#' df <- toDF(rdd)
245243
#'}
@@ -261,8 +259,7 @@ setMethod("toDF", signature(x = "RDD"),
261259
#' @export
262260
#' @examples
263261
#'\dontrun{
264-
#' sc <- sparkR.init()
265-
#' sqlContext <- sparkRSQL.init(sc)
262+
#' sparkR.session()
266263
#' path <- "path/to/file.json"
267264
#' df <- read.json(path)
268265
#' df <- jsonFile(path)
@@ -309,8 +306,7 @@ jsonFile <- function(x, ...) {
309306
#' @noRd
310307
#' @examples
311308
#'\dontrun{
312-
#' sc <- sparkR.init()
313-
#' sqlContext <- sparkRSQL.init(sc)
309+
#' sparkR.session()
314310
#' rdd <- texFile(sc, "path/to/json")
315311
#' df <- jsonRDD(sqlContext, rdd)
316312
#'}
@@ -401,8 +397,7 @@ parquetFile <- function(x, ...) {
401397
#' @export
402398
#' @examples
403399
#'\dontrun{
404-
#' sc <- sparkR.init()
405-
#' sqlContext <- sparkRSQL.init(sc)
400+
#' sparkR.session()
406401
#' path <- "path/to/file.txt"
407402
#' df <- read.text(path)
408403
#' }
@@ -432,8 +427,7 @@ read.text <- function(x, ...) {
432427
#' @export
433428
#' @examples
434429
#'\dontrun{
435-
#' sc <- sparkR.init()
436-
#' sqlContext <- sparkRSQL.init(sc)
430+
#' sparkR.session()
437431
#' path <- "path/to/file.json"
438432
#' df <- read.json(path)
439433
#' createOrReplaceTempView(df, "table")
@@ -464,8 +458,7 @@ sql <- function(x, ...) {
464458
#' @export
465459
#' @examples
466460
#'\dontrun{
467-
#' sc <- sparkR.init()
468-
#' sqlContext <- sparkRSQL.init(sc)
461+
#' sparkR.session()
469462
#' path <- "path/to/file.json"
470463
#' df <- read.json(path)
471464
#' createOrReplaceTempView(df, "table")
@@ -489,8 +482,7 @@ tableToDF <- function(tableName) {
489482
#' @export
490483
#' @examples
491484
#'\dontrun{
492-
#' sc <- sparkR.init()
493-
#' sqlContext <- sparkRSQL.init(sc)
485+
#' sparkR.session()
494486
#' tables("hive")
495487
#' }
496488
#' @name tables
@@ -516,8 +508,7 @@ tables <- function(x, ...) {
516508
#' @export
517509
#' @examples
518510
#'\dontrun{
519-
#' sc <- sparkR.init()
520-
#' sqlContext <- sparkRSQL.init(sc)
511+
#' sparkR.session()
521512
#' tableNames("hive")
522513
#' }
523514
#' @name tableNames
@@ -545,8 +536,7 @@ tableNames <- function(x, ...) {
545536
#' @export
546537
#' @examples
547538
#'\dontrun{
548-
#' sc <- sparkR.init()
549-
#' sqlContext <- sparkRSQL.init(sc)
539+
#' sparkR.session()
550540
#' path <- "path/to/file.json"
551541
#' df <- read.json(path)
552542
#' createOrReplaceTempView(df, "table")
@@ -575,8 +565,7 @@ cacheTable <- function(x, ...) {
575565
#' @export
576566
#' @examples
577567
#'\dontrun{
578-
#' sc <- sparkR.init()
579-
#' sqlContext <- sparkRSQL.init(sc)
568+
#' sparkR.session()
580569
#' path <- "path/to/file.json"
581570
#' df <- read.json(path)
582571
#' createOrReplaceTempView(df, "table")
@@ -629,8 +618,7 @@ clearCache <- function() {
629618
#' @export
630619
#' @examples
631620
#' \dontrun{
632-
#' sc <- sparkR.init()
633-
#' sqlContext <- sparkRSQL.init(sc)
621+
#' sparkR.session()
634622
#' df <- read.df(path, "parquet")
635623
#' createOrReplaceTempView(df, "table")
636624
#' dropTempTable("table")
@@ -694,8 +682,7 @@ dropTempView <- function(viewName) {
694682
#' @export
695683
#' @examples
696684
#'\dontrun{
697-
#' sc <- sparkR.init()
698-
#' sqlContext <- sparkRSQL.init(sc)
685+
#' sparkR.session()
699686
#' df1 <- read.df("path/to/file.json", source = "json")
700687
#' schema <- structType(structField("name", "string"),
701688
#' structField("info", "map<string,double>"))
@@ -758,9 +745,8 @@ loadDF <- function(x, ...) {
758745
#' @export
759746
#' @examples
760747
#'\dontrun{
761-
#' sc <- sparkR.init()
762-
#' sqlContext <- sparkRSQL.init(sc)
763-
#' df <- sparkRSQL.createExternalTable("myjson", path="path/to/json", source="json")
748+
#' sparkR.session()
749+
#' df <- createExternalTable("myjson", path="path/to/json", source="json")
764750
#' }
765751
#' @name createExternalTable
766752
#' @method createExternalTable default
@@ -806,8 +792,7 @@ createExternalTable <- function(x, ...) {
806792
#' @export
807793
#' @examples
808794
#'\dontrun{
809-
#' sc <- sparkR.init()
810-
#' sqlContext <- sparkRSQL.init(sc)
795+
#' sparkR.session()
811796
#' jdbcUrl <- "jdbc:mysql://localhost:3306/databasename"
812797
#' df <- read.jdbc(jdbcUrl, "table", predicates = list("field<=123"), user = "username")
813798
#' df2 <- read.jdbc(jdbcUrl, "table2", partitionColumn = "index", lowerBound = 0,

‎R/pkg/R/mllib.R‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,7 @@ setClass("KMeansModel", representation(jobj = "jobj"))
6767
#' @export
6868
#' @examples
6969
#' \dontrun{
70-
#' sc <- sparkR.init()
71-
#' sqlContext <- sparkRSQL.init(sc)
70+
#' sparkR.session()
7271
#' data(iris)
7372
#' df <- createDataFrame(iris)
7473
#' model <- spark.glm(df, Sepal_Length ~ Sepal_Width, family="gaussian")
@@ -115,8 +114,7 @@ setMethod(
115114
#' @export
116115
#' @examples
117116
#' \dontrun{
118-
#' sc <- sparkR.init()
119-
#' sqlContext <- sparkRSQL.init(sc)
117+
#' sparkR.session()
120118
#' data(iris)
121119
#' df <- createDataFrame(iris)
122120
#' model <- glm(Sepal_Length ~ Sepal_Width, df, family="gaussian")

‎R/pkg/R/schema.R‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@
2929
#' @export
3030
#' @examples
3131
#'\dontrun{
32-
#' sc <- sparkR.init()
33-
#' sqlCtx <- sparkRSQL.init(sc)
34-
#' rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) })
35-
#' schema <- structType(structField("a", "integer"), structField("b", "string"))
36-
#' df <- createDataFrame(sqlCtx, rdd, schema)
32+
#' schema <- structType(structField("a", "integer"), structField("c", "string"),
33+
#' structField("avg", "double"))
34+
#' df1 <- gapply(df, list("a", "c"),
35+
#' function(key, x) { y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE) },
36+
#' schema)
3737
#' }
3838
structType <- function(x, ...) {
3939
UseMethod("structType", x)
@@ -90,13 +90,13 @@ print.structType <- function(x, ...) {
9090
#' @export
9191
#' @examples
9292
#'\dontrun{
93-
#' sc <- sparkR.init()
94-
#' sqlCtx <- sparkRSQL.init(sc)
95-
#' rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) })
96-
#' field1 <- structField("a", "integer", TRUE)
97-
#' field2 <- structField("b", "string", TRUE)
98-
#' schema <- structType(field1, field2)
99-
#' df <- createDataFrame(sqlCtx, rdd, schema)
93+
#' field1 <- structField("a", "integer")
94+
#' field2 <- structField("c", "string")
95+
#' field3 <- structField("avg", "double")
96+
#' schema <- structType(field1, field2, field3)
97+
#' df1 <- gapply(df, list("a", "c"),
98+
#' function(key, x) { y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE) },
99+
#' schema)
100100
#' }
101101

102102
structField <- function(x, ...) {

‎R/pkg/R/sparkR.R‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,7 @@ sparkR.session.stop <- function() {
8989

9090
#' (Deprecated) Initialize a new Spark Context.
9191
#'
92-
#' This function initializes a new SparkContext. For details on how to initialize
93-
#' and use SparkR, refer to SparkR programming guide at
94-
#' \url{http://spark.apache.org/docs/latest/sparkr.html#starting-up-sparkcontext-sqlcontext}.
92+
#' This function initializes a new SparkContext.
9593
#'
9694
#' @param master The Spark master URL
9795
#' @param appName Application name to register with cluster manager
@@ -312,6 +310,9 @@ sparkRHive.init <- function(jsc = NULL) {
312310
#' Additional Spark properties can be set (...), and these named parameters take priority over
313311
#' over values in master, appName, named lists of sparkConfig.
314312
#'
313+
#' For details on how to initialize and use SparkR, refer to SparkR programming guide at
314+
#' \url{http://spark.apache.org/docs/latest/sparkr.html#starting-up-sparksession}.
315+
#'
315316
#' @param master The Spark master URL
316317
#' @param appName Application name to register with cluster manager
317318
#' @param sparkHome Spark Home directory

0 commit comments

Comments
 (0)