From 524e0aaede8338027571364e193b5b257c28d76e Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Sun, 28 Jun 2015 23:09:21 -0700 Subject: [PATCH 1/2] Add option function to df reader and writer --- python/pyspark/sql/readwriter.py | 14 ++++++++++++++ python/pyspark/sql/tests.py | 1 + 2 files changed, 15 insertions(+) diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py index 1b7bc0f9a12b..110d7db656a0 100644 --- a/python/pyspark/sql/readwriter.py +++ b/python/pyspark/sql/readwriter.py @@ -73,6 +73,13 @@ def schema(self, schema): self._jreader = self._jreader.schema(jschema) return self + @since(1.4) + def option(self, key, value): + """Adds an input option for the underlying data source. + """ + self._jreader = self._jreader.option(key, value) + return self + @since(1.4) def options(self, **options): """Adds input options for the underlying data source. @@ -235,6 +242,13 @@ def format(self, source): self._jwrite = self._jwrite.format(source) return self + @since(1.4) + def option(self, key, value): + """Adds an output option for the underlying data source. + """ + self._jwrite = self._jwrite.option(key, value) + return self + @since(1.4) def options(self, **options): """Adds output options for the underlying data source. diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index e6a434e4b2df..ffee43a94bab 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -564,6 +564,7 @@ def test_save_and_load_builder(self): self.assertEqual(sorted(df.collect()), sorted(actual.collect())) df.write.mode("overwrite").options(noUse="this options will not be used in save.")\ + .option("noUse", "this option will not be used in save.")\ .format("json").save(path=tmpPath) actual =\ self.sqlCtx.read.format("json")\ From c63d419f10e1bf8efd7228206c032739b90d24c9 Mon Sep 17 00:00:00 2001 From: Cheolsoo Park Date: Sun, 28 Jun 2015 23:44:30 -0700 Subject: [PATCH 2/2] Fix version --- python/pyspark/sql/readwriter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py index 110d7db656a0..c4cc62e82a16 100644 --- a/python/pyspark/sql/readwriter.py +++ b/python/pyspark/sql/readwriter.py @@ -73,7 +73,7 @@ def schema(self, schema): self._jreader = self._jreader.schema(jschema) return self - @since(1.4) + @since(1.5) def option(self, key, value): """Adds an input option for the underlying data source. """ @@ -242,7 +242,7 @@ def format(self, source): self._jwrite = self._jwrite.format(source) return self - @since(1.4) + @since(1.5) def option(self, key, value): """Adds an output option for the underlying data source. """