Skip to content

Commit a5c0343

Browse files
committed
LDA should support disable checkpoint
1 parent b66b97c commit a5c0343

File tree

1 file changed

+11
-5
lines changed

1 file changed

+11
-5
lines changed

mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,11 @@ final class EMLDAOptimizer extends LDAOptimizer {
142142
this.k = k
143143
this.vocabSize = docs.take(1).head._2.size
144144
this.checkpointInterval = lda.getCheckpointInterval
145-
this.graphCheckpointer = new PeriodicGraphCheckpointer[TopicCounts, TokenCount](
146-
checkpointInterval, graph.vertices.sparkContext)
147-
this.graphCheckpointer.update(this.graph)
145+
if (this.checkpointInterval != -1) {
146+
this.graphCheckpointer = new PeriodicGraphCheckpointer[TopicCounts, TokenCount](
147+
checkpointInterval, graph.vertices.sparkContext)
148+
this.graphCheckpointer.update(this.graph)
149+
}
148150
this.globalTopicTotals = computeGlobalTopicTotals()
149151
this
150152
}
@@ -189,7 +191,9 @@ final class EMLDAOptimizer extends LDAOptimizer {
189191
// Update the vertex descriptors with the new counts.
190192
val newGraph = Graph(docTopicDistributions, graph.edges)
191193
graph = newGraph
192-
graphCheckpointer.update(newGraph)
194+
if (this.checkpointInterval != -1) {
195+
graphCheckpointer.update(newGraph)
196+
}
193197
globalTopicTotals = computeGlobalTopicTotals()
194198
this
195199
}
@@ -208,7 +212,9 @@ final class EMLDAOptimizer extends LDAOptimizer {
208212

209213
override private[clustering] def getLDAModel(iterationTimes: Array[Double]): LDAModel = {
210214
require(graph != null, "graph is null, EMLDAOptimizer not initialized.")
211-
this.graphCheckpointer.deleteAllCheckpoints()
215+
if (this.checkpointInterval != -1) {
216+
this.graphCheckpointer.deleteAllCheckpoints()
217+
}
212218
// The constructor's default arguments assume gammaShape = 100 to ensure equivalence in
213219
// LDAModel.toLocal conversion
214220
new DistributedLDAModel(this.graph, this.globalTopicTotals, this.k, this.vocabSize,

0 commit comments

Comments
 (0)