Skip to content

[SPARK-12300][SQL][PYSPARK] fix schema inferance on local collections #10275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

[SPARK-12300][SQL][PYSPARK] fix schema inferance on local collections #10275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
935c167
Look at all elements of the local collection when infering the types
holdenk Dec 12, 2015
7ca7eca
Add a test
holdenk Dec 12, 2015
e01cbb4
Merge branch 'master' into SPARK-12300-fix-schmea-inferance-on-local-…
holdenk Dec 12, 2015
6e463fd
Fix test
holdenk Dec 12, 2015
b863cc9
python 3 import reduce from functools
holdenk Dec 12, 2015
3f2b825
Merge branch 'master' into SPARK-12300-fix-schmea-inferance-on-local-…
holdenk Dec 30, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions python/pyspark/sql/context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import sys
import warnings
import json
from functools import reduce

if sys.version >= '3':
basestring = unicode = str
Expand Down Expand Up @@ -236,14 +237,9 @@ def _inferSchemaFromList(self, data):
if type(first) is dict:
warnings.warn("inferring schema from dict is deprecated,"
"please use pyspark.sql.Row instead")
schema = _infer_schema(first)
schema = reduce(_merge_type, map(_infer_schema, data))
if _has_nulltype(schema):
for r in data:
schema = _merge_type(schema, _infer_schema(r))
if not _has_nulltype(schema):
break
else:
raise ValueError("Some of types cannot be determined after inferring")
raise ValueError("Some of types cannot be determined after inferring")
return schema

def _inferSchema(self, rdd, samplingRatio=None):
Expand Down
11 changes: 11 additions & 0 deletions python/pyspark/sql/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,17 @@ def test_apply_schema_to_row(self):
df3 = self.sqlCtx.createDataFrame(rdd, df.schema)
self.assertEqual(10, df3.count())

def test_infer_schema_to_local(self):
input = [{"a": 1}, {"b": "coffee"}]
rdd = self.sc.parallelize(input)
df = self.sqlCtx.createDataFrame(input)
df2 = self.sqlCtx.createDataFrame(rdd, samplingRatio=1.0)
self.assertEqual(df.schema, df2.schema)

rdd = self.sc.parallelize(range(10)).map(lambda x: Row(a=x))
df3 = self.sqlCtx.createDataFrame(rdd, df.schema)
self.assertEqual(10, df3.count())

def test_serialize_nested_array_and_map(self):
d = [Row(l=[Row(a=1, b='s')], d={"key": Row(c=1.0, d="2")})]
rdd = self.sc.parallelize(d)
Expand Down