Skip to content

Commit d2bd26c

Browse files
hvanhovellhvanhovell
committed
Merge remote-tracking branch 'apache-github/master' into SPARK-17106
# Conflicts: # sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/subquery.scala
2 parents 8f11d35 + 0f6aa8a commit d2bd26c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+428
-376
lines changed

‎R/pkg/R/RDD.R‎

Lines changed: 50 additions & 50 deletions
Large diffs are not rendered by default.

‎R/pkg/R/SQLContext.R‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ createDataFrame.default <- function(data, schema = NULL, samplingRatio = 1.0) {
218218
}
219219

220220
if (is.null(schema) || (!inherits(schema, "structType") && is.null(names(schema)))) {
221-
row <- first(rdd)
221+
row <- firstRDD(rdd)
222222
names <- if (is.null(schema)) {
223223
names(row)
224224
} else {

‎R/pkg/R/context.R‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ spark.lapply <- function(list, func) {
267267
sc <- getSparkContext()
268268
rdd <- parallelize(sc, list, length(list))
269269
results <- map(rdd, func)
270-
local <- collect(results)
270+
local <- collectRDD(results)
271271
local
272272
}
273273

‎R/pkg/R/generics.R‎

Lines changed: 58 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,7 @@
2323
setGeneric("aggregateRDD",
2424
function(x, zeroValue, seqOp, combOp) { standardGeneric("aggregateRDD") })
2525

26-
# @rdname cache-methods
27-
# @export
28-
setGeneric("cache", function(x) { standardGeneric("cache") })
26+
setGeneric("cacheRDD", function(x) { standardGeneric("cacheRDD") })
2927

3028
# @rdname coalesce
3129
# @seealso repartition
@@ -36,9 +34,7 @@ setGeneric("coalesce", function(x, numPartitions, ...) { standardGeneric("coales
3634
# @export
3735
setGeneric("checkpoint", function(x) { standardGeneric("checkpoint") })
3836

39-
# @rdname collect-methods
40-
# @export
41-
setGeneric("collect", function(x, ...) { standardGeneric("collect") })
37+
setGeneric("collectRDD", function(x, ...) { standardGeneric("collectRDD") })
4238

4339
# @rdname collect-methods
4440
# @export
@@ -51,9 +47,9 @@ setGeneric("collectPartition",
5147
standardGeneric("collectPartition")
5248
})
5349

54-
# @rdname nrow
55-
# @export
56-
setGeneric("count", function(x) { standardGeneric("count") })
50+
setGeneric("countRDD", function(x) { standardGeneric("countRDD") })
51+
52+
setGeneric("lengthRDD", function(x) { standardGeneric("lengthRDD") })
5753

5854
# @rdname countByValue
5955
# @export
@@ -74,17 +70,13 @@ setGeneric("approxQuantile",
7470
standardGeneric("approxQuantile")
7571
})
7672

77-
# @rdname distinct
78-
# @export
79-
setGeneric("distinct", function(x, numPartitions = 1) { standardGeneric("distinct") })
73+
setGeneric("distinctRDD", function(x, numPartitions = 1) { standardGeneric("distinctRDD") })
8074

8175
# @rdname filterRDD
8276
# @export
8377
setGeneric("filterRDD", function(x, f) { standardGeneric("filterRDD") })
8478

85-
# @rdname first
86-
# @export
87-
setGeneric("first", function(x, ...) { standardGeneric("first") })
79+
setGeneric("firstRDD", function(x, ...) { standardGeneric("firstRDD") })
8880

8981
# @rdname flatMap
9082
# @export
@@ -110,6 +102,8 @@ setGeneric("glom", function(x) { standardGeneric("glom") })
110102
# @export
111103
setGeneric("histogram", function(df, col, nbins=10) { standardGeneric("histogram") })
112104

105+
setGeneric("joinRDD", function(x, y, ...) { standardGeneric("joinRDD") })
106+
113107
# @rdname keyBy
114108
# @export
115109
setGeneric("keyBy", function(x, func) { standardGeneric("keyBy") })
@@ -152,9 +146,7 @@ setGeneric("getNumPartitions", function(x) { standardGeneric("getNumPartitions")
152146
# @export
153147
setGeneric("numPartitions", function(x) { standardGeneric("numPartitions") })
154148

155-
# @rdname persist
156-
# @export
157-
setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })
149+
setGeneric("persistRDD", function(x, newLevel) { standardGeneric("persistRDD") })
158150

159151
# @rdname pipeRDD
160152
# @export
@@ -168,10 +160,7 @@ setGeneric("pivot", function(x, colname, values = list()) { standardGeneric("piv
168160
# @export
169161
setGeneric("reduce", function(x, func) { standardGeneric("reduce") })
170162

171-
# @rdname repartition
172-
# @seealso coalesce
173-
# @export
174-
setGeneric("repartition", function(x, ...) { standardGeneric("repartition") })
163+
setGeneric("repartitionRDD", function(x, ...) { standardGeneric("repartitionRDD") })
175164

176165
# @rdname sampleRDD
177166
# @export
@@ -193,16 +182,16 @@ setGeneric("saveAsTextFile", function(x, path) { standardGeneric("saveAsTextFile
193182
# @export
194183
setGeneric("setName", function(x, name) { standardGeneric("setName") })
195184

185+
setGeneric("showRDD", function(object, ...) { standardGeneric("showRDD") })
186+
196187
# @rdname sortBy
197188
# @export
198189
setGeneric("sortBy",
199190
function(x, func, ascending = TRUE, numPartitions = 1) {
200191
standardGeneric("sortBy")
201192
})
202193

203-
# @rdname take
204-
# @export
205-
setGeneric("take", function(x, num) { standardGeneric("take") })
194+
setGeneric("takeRDD", function(x, num) { standardGeneric("takeRDD") })
206195

207196
# @rdname takeOrdered
208197
# @export
@@ -223,9 +212,7 @@ setGeneric("top", function(x, num) { standardGeneric("top") })
223212
# @export
224213
setGeneric("unionRDD", function(x, y) { standardGeneric("unionRDD") })
225214

226-
# @rdname unpersist-methods
227-
# @export
228-
setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") })
215+
setGeneric("unpersistRDD", function(x, ...) { standardGeneric("unpersistRDD") })
229216

230217
# @rdname zipRDD
231218
# @export
@@ -343,9 +330,7 @@ setGeneric("join", function(x, y, ...) { standardGeneric("join") })
343330
# @export
344331
setGeneric("leftOuterJoin", function(x, y, numPartitions) { standardGeneric("leftOuterJoin") })
345332

346-
#' @rdname partitionBy
347-
#' @export
348-
setGeneric("partitionBy", function(x, ...) { standardGeneric("partitionBy") })
333+
setGeneric("partitionByRDD", function(x, ...) { standardGeneric("partitionByRDD") })
349334

350335
# @rdname reduceByKey
351336
# @seealso groupByKey
@@ -414,6 +399,14 @@ setGeneric("as.data.frame",
414399
#' @export
415400
setGeneric("attach")
416401

402+
#' @rdname cache
403+
#' @export
404+
setGeneric("cache", function(x) { standardGeneric("cache") })
405+
406+
#' @rdname collect
407+
#' @export
408+
setGeneric("collect", function(x, ...) { standardGeneric("collect") })
409+
417410
#' @rdname columns
418411
#' @export
419412
setGeneric("colnames", function(x, do.NULL = TRUE, prefix = "col") { standardGeneric("colnames") })
@@ -434,6 +427,10 @@ setGeneric("coltypes<-", function(x, value) { standardGeneric("coltypes<-") })
434427
#' @export
435428
setGeneric("columns", function(x) {standardGeneric("columns") })
436429

430+
#' @rdname nrow
431+
#' @export
432+
setGeneric("count", function(x) { standardGeneric("count") })
433+
437434
#' @rdname cov
438435
#' @export
439436
setGeneric("cov", function(x, ...) {standardGeneric("cov") })
@@ -477,6 +474,10 @@ setGeneric("gapplyCollect", function(x, ...) { standardGeneric("gapplyCollect")
477474
#' @export
478475
setGeneric("describe", function(x, col, ...) { standardGeneric("describe") })
479476

477+
#' @rdname distinct
478+
#' @export
479+
setGeneric("distinct", function(x) { standardGeneric("distinct") })
480+
480481
#' @rdname drop
481482
#' @export
482483
setGeneric("drop", function(x, ...) { standardGeneric("drop") })
@@ -519,6 +520,10 @@ setGeneric("fillna", function(x, value, cols = NULL) { standardGeneric("fillna")
519520
#' @export
520521
setGeneric("filter", function(x, condition) { standardGeneric("filter") })
521522

523+
#' @rdname first
524+
#' @export
525+
setGeneric("first", function(x, ...) { standardGeneric("first") })
526+
522527
#' @rdname groupBy
523528
#' @export
524529
setGeneric("group_by", function(x, ...) { standardGeneric("group_by") })
@@ -555,17 +560,25 @@ setGeneric("mutate", function(.data, ...) {standardGeneric("mutate") })
555560
#' @export
556561
setGeneric("orderBy", function(x, col, ...) { standardGeneric("orderBy") })
557562

563+
#' @rdname persist
564+
#' @export
565+
setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })
566+
558567
#' @rdname printSchema
559568
#' @export
560569
setGeneric("printSchema", function(x) { standardGeneric("printSchema") })
561570

571+
#' @rdname registerTempTable-deprecated
572+
#' @export
573+
setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
574+
562575
#' @rdname rename
563576
#' @export
564577
setGeneric("rename", function(x, ...) { standardGeneric("rename") })
565578

566-
#' @rdname registerTempTable-deprecated
579+
#' @rdname repartition
567580
#' @export
568-
setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
581+
setGeneric("repartition", function(x, ...) { standardGeneric("repartition") })
569582

570583
#' @rdname sample
571584
#' @export
@@ -592,6 +605,10 @@ setGeneric("saveAsTable", function(df, tableName, source = NULL, mode = "error",
592605
#' @export
593606
setGeneric("str")
594607

608+
#' @rdname take
609+
#' @export
610+
setGeneric("take", function(x, num) { standardGeneric("take") })
611+
595612
#' @rdname mutate
596613
#' @export
597614
setGeneric("transform", function(`_data`, ...) {standardGeneric("transform") })
@@ -674,6 +691,10 @@ setGeneric("union", function(x, y) { standardGeneric("union") })
674691
#' @export
675692
setGeneric("unionAll", function(x, y) { standardGeneric("unionAll") })
676693

694+
#' @rdname unpersist-methods
695+
#' @export
696+
setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") })
697+
677698
#' @rdname filter
678699
#' @export
679700
setGeneric("where", function(x, condition) { standardGeneric("where") })
@@ -771,6 +792,10 @@ setGeneric("over", function(x, window) { standardGeneric("over") })
771792

772793
###################### WindowSpec Methods ##########################
773794

795+
#' @rdname partitionBy
796+
#' @export
797+
setGeneric("partitionBy", function(x, ...) { standardGeneric("partitionBy") })
798+
774799
#' @rdname rowsBetween
775800
#' @export
776801
setGeneric("rowsBetween", function(x, start, end) { standardGeneric("rowsBetween") })

‎R/pkg/R/mllib.R‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ setMethod("spark.glm", signature(data = "SparkDataFrame", formula = "formula"),
140140

141141
jobj <- callJStatic("org.apache.spark.ml.r.GeneralizedLinearRegressionWrapper",
142142
"fit", formula, data@sdf, family$family, family$link,
143-
tol, as.integer(maxIter), weightCol)
143+
tol, as.integer(maxIter), as.character(weightCol))
144144
return(new("GeneralizedLinearRegressionModel", jobj = jobj))
145145
})
146146

0 commit comments

Comments
 (0)