Skip to content

Commit 59b5562

Browse files
committed
[SQL] localCheckpoint() adding no-param declaration
1 parent aa70ce6 commit 59b5562

File tree

1 file changed

+25
-0
lines changed

1 file changed

+25
-0
lines changed

sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,18 @@ class Dataset[T] private[sql](
513513
@InterfaceStability.Evolving
514514
def isStreaming: Boolean = logicalPlan.isStreaming
515515

516+
/**
517+
* Eagerly checkpoint a Dataset and return the new Dataset. Checkpointing can be used to truncate
518+
* the logical plan of this Dataset, which is especially useful in iterative algorithms where the
519+
* plan may grow exponentially. It will be saved to files inside the checkpoint
520+
* directory set with `SparkContext#setCheckpointDir`.
521+
* @group basic
522+
* @since 2.1.0
523+
*/
524+
@Experimental
525+
@InterfaceStability.Evolving
526+
def checkpoint(): Dataset[T] = _checkpoint(eager = true)
527+
516528
/**
517529
* Returns a checkpointed version of this Dataset. Checkpointing can be used to truncate the
518530
* logical plan of this Dataset, which is especially useful in iterative algorithms where the
@@ -526,6 +538,19 @@ class Dataset[T] private[sql](
526538
@InterfaceStability.Evolving
527539
def checkpoint(eager: Boolean = true): Dataset[T] = _checkpoint(eager = eager)
528540

541+
/**
542+
* Eagerly locally checkpoints a Dataset and return the new Dataset. Checkpointing can be
543+
* used to truncate the logical plan of this Dataset, which is especially useful in iterative
544+
* algorithms where the plan may grow exponentially. Local checkpoints are written to executor
545+
* storage and despite potentially faster they are unreliable and may compromise job completion.
546+
*
547+
* @group basic
548+
* @since 2.3.0
549+
*/
550+
@Experimental
551+
@InterfaceStability.Evolving
552+
def localCheckpoint(): Dataset[T] = _checkpoint(eager = true, local = true)
553+
529554
/**
530555
* Locally checkpoints a Dataset and return the new Dataset. Checkpointing can be used to truncate
531556
* the logical plan of this Dataset, which is especially useful in iterative algorithms where the

0 commit comments

Comments
 (0)