From 8d87c0f2bd9140928915f835fd7d21b178422c69 Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Tue, 7 Jun 2016 13:04:52 -0700 Subject: [PATCH 1/2] update python sql interface refreshTable --- python/pyspark/sql/catalog.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py index 3033f147bc96..6760f5d8680c 100644 --- a/python/pyspark/sql/catalog.py +++ b/python/pyspark/sql/catalog.py @@ -232,6 +232,11 @@ def clearCache(self): """Removes all cached tables from the in-memory cache.""" self._jcatalog.clearCache() + @since(2.0) + def refreshTable(self, tableName): + """Invalidate and refresh all the cached the metadata of the given table.""" + self._jcatalog.refreshTable(tableName) + def _reset(self): """(Internal use only) Drop all existing databases (except "default"), tables, partitions and functions, and set the current database to "default". From 87fcb7c6060aa6a5de3d208daffd7581d302c5ff Mon Sep 17 00:00:00 2001 From: WeichenXu Date: Thu, 16 Jun 2016 05:51:31 -0700 Subject: [PATCH 2/2] update comment --- python/pyspark/sql/catalog.py | 2 +- .../src/main/scala/org/apache/spark/sql/catalog/Catalog.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py index 6760f5d8680c..4af930a3cd56 100644 --- a/python/pyspark/sql/catalog.py +++ b/python/pyspark/sql/catalog.py @@ -234,7 +234,7 @@ def clearCache(self): @since(2.0) def refreshTable(self, tableName): - """Invalidate and refresh all the cached the metadata of the given table.""" + """Invalidate and refresh all the cached metadata of the given table.""" self._jcatalog.refreshTable(tableName) def _reset(self): diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala index 6ddb1a7a1f1a..7985f2f7cdae 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala @@ -214,7 +214,7 @@ abstract class Catalog { def clearCache(): Unit /** - * Invalidate and refresh all the cached the metadata of the given table. For performance reasons, + * Invalidate and refresh all the cached metadata of the given table. For performance reasons, * Spark SQL or the external data source library it uses might cache certain metadata about a * table, such as the location of blocks. When those change outside of Spark SQL, users should * call this function to invalidate the cache.