Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ private[ui] class PoolPage(parent: JobProgressTab) extends WebUIPage("pool") {
case Some(s) => s.values.toSeq
case None => Seq[StageInfo]()
}

val activeStagesTable =
new StageTableBase(activeStages.sortBy(_.submissionTime).reverse, parent)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,10 @@ object TwitterAlgebirdCMS {
if (rdd.count() != 0) {
val partial = rdd.first()
val partialTopK = partial.heavyHitters.map(id =>
(id, partial.frequency(id).estimate)).toSeq.sortBy(_._2).reverse.slice(0, TOPK)
(id, partial.frequency(id).estimate)).toSeq.sortBy(_._2).reverse.take(TOPK)
globalCMS ++= partial
val globalTopK = globalCMS.heavyHitters.map(id =>
(id, globalCMS.frequency(id).estimate)).toSeq.sortBy(_._2).reverse.slice(0, TOPK)
(id, globalCMS.frequency(id).estimate)).toSeq.sortBy(_._2).reverse.take(TOPK)
println("Approx heavy hitters at %2.2f%% threshold this batch: %s".format(PERC,
partialTopK.mkString("[", ",", "]")))
println("Approx heavy hitters at %2.2f%% threshold overall: %s".format(PERC,
Expand All @@ -102,7 +102,7 @@ object TwitterAlgebirdCMS {
{case (id, count) => (count, id)})
.sortByKey(ascending = false).take(TOPK)
globalExact = mm.plus(globalExact.toMap, partialMap)
val globalTopK = globalExact.toSeq.sortBy(_._2).reverse.slice(0, TOPK)
val globalTopK = globalExact.toSeq.sortBy(_._2).reverse.take(TOPK)
println("Exact heavy hitters this batch: %s".format(partialTopK.mkString("[", ",", "]")))
println("Exact heavy hitters overall: %s".format(globalTopK.mkString("[", ",", "]")))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ abstract class GeneralizedLinearAlgorithm[M <: GeneralizedLinearModel]
val intercept = if (addIntercept) weightsWithIntercept(weightsWithIntercept.size - 1) else 0.0
val weights =
if (addIntercept) {
Vectors.dense(weightsWithIntercept.toArray.slice(0, weightsWithIntercept.size - 1))
Vectors.dense(weightsWithIntercept.toArray.tail)
} else {
weightsWithIntercept
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ object MFDataGenerator {
val mn = m * n
val shuffled = rand.shuffle(1 to mn toList)

val omega = shuffled.slice(0, sampSize)
val omega = shuffled.take(sampSize)
val ordered = omega.sortWith(_ < _).toArray
val trainData: RDD[(Int, Int, Double)] = sc.parallelize(ordered)
.map(x => (fullData.indexRows(x - 1), fullData.indexColumns(x - 1), fullData.get(x - 1)))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ case class RangePartitioning(ordering: Seq[SortOrder], numPartitions: Int)
override def satisfies(required: Distribution): Boolean = required match {
case UnspecifiedDistribution => true
case OrderedDistribution(requiredOrdering) =>
val minSize = Seq(requiredOrdering.size, ordering.size).min
val minSize = math.min(requiredOrdering.size, ordering.size)
requiredOrdering.take(minSize) == ordering.take(minSize)
case ClusteredDistribution(requiredClustering) =>
clusteringSet.subsetOf(requiredClustering.toSet)
Expand Down