|
35 | 35 | from pyspark.rdd import RDD, SchemaRDD |
36 | 36 |
|
37 | 37 | from py4j.java_collections import ListConverter |
| 38 | +from py4j.protocol import Py4JError |
38 | 39 |
|
39 | 40 |
|
40 | 41 | class SparkContext(object): |
@@ -621,31 +622,38 @@ class HiveContext(SQLContext): |
621 | 622 |
|
622 | 623 | @property |
623 | 624 | def _ssql_ctx(self): |
624 | | - if not hasattr(self, '_scala_HiveContext'): |
625 | | - self._scala_HiveContext = self._jvm.HiveContext(self._jsc.sc()) |
626 | | - return self._scala_HiveContext |
| 625 | + try: |
| 626 | + if not hasattr(self, '_scala_HiveContext'): |
| 627 | + self._scala_HiveContext = self._get_hive_ctx() |
| 628 | + return self._scala_HiveContext |
| 629 | + except Py4JError as e: |
| 630 | + raise Exception("You must build Spark with Hive. Export 'SPARK_HIVE=true' and run " \ |
| 631 | + "sbt/sbt assembly" , e) |
| 632 | + |
| 633 | + def _get_hive_ctx(self): |
| 634 | + return self._jvm.HiveContext(self._jsc.sc()) |
627 | 635 |
|
628 | 636 | def hiveql(self, hqlQuery): |
| 637 | + """ |
| 638 | + Runs a query expressed in HiveQL, returning the result as a L{SchemaRDD}. |
| 639 | + """ |
629 | 640 | return SchemaRDD(self._ssql_ctx.hiveql(hqlQuery), self) |
630 | 641 |
|
631 | 642 | def hql(self, hqlQuery): |
| 643 | + """ |
| 644 | + Runs a query expressed in HiveQL, returning the result as a L{SchemaRDD}. |
| 645 | + """ |
632 | 646 | return self.hiveql(hqlQuery) |
633 | 647 |
|
634 | 648 | class LocalHiveContext(HiveContext): |
635 | 649 |
|
636 | | - @property |
637 | | - def _ssql_ctx(self): |
638 | | - if not hasattr(self, '_scala_LocalHiveContext'): |
639 | | - self._scala_LocalHiveContext = self._jvm.LocalHiveContext(self._jsc.sc()) |
640 | | - return self._scala_LocalHiveContext |
| 650 | + def _get_hive_ctx(self): |
| 651 | + return self._jvm.LocalHiveContext(self._jsc.sc()) |
641 | 652 |
|
642 | 653 | class TestHiveContext(HiveContext): |
643 | 654 |
|
644 | | - @property |
645 | | - def _ssql_ctx(self): |
646 | | - if not hasattr(self, '_scala_TestHiveContext'): |
647 | | - self._scala_TestHiveContext = self._jvm.TestHiveContext(self._jsc.sc()) |
648 | | - return self._scala_TestHiveContext |
| 655 | + def _get_hive_ctx(self): |
| 656 | + return self._jvm.TestHiveContext(self._jsc.sc()) |
649 | 657 |
|
650 | 658 | def _test(): |
651 | 659 | import atexit |
|
0 commit comments