Merge pull request #7581 from jorisvandenbossche/sql-reflect-database

SQL: don't reflect full database GH7396

Author: jorisvandenbossche

pandas/io/sql.py

@@ -76,6 +76,17 @@ def _parse_date_columns(data_frame, parse_dates):
     return data_frame
 
 
+def _is_sqlalchemy_engine(con):
+    try:
+        import sqlalchemy
+        if isinstance(con, sqlalchemy.engine.Engine):
+            return True
+        else:
+            return False
+    except ImportError:
+        return False
+
+
 def execute(sql, con, cur=None, params=None):
     """
     Execute the given SQL query using the provided connection object.
@@ -262,7 +273,15 @@ def read_sql_table(table_name, con, index_col=None, coerce_float=True,
 
 
     """
-    pandas_sql = PandasSQLAlchemy(con)
+    import sqlalchemy
+    from sqlalchemy.schema import MetaData
+    meta = MetaData(con)
+    try:
+        meta.reflect(only=[table_name])
+    except sqlalchemy.exc.InvalidRequestError:
+        raise ValueError("Table %s not found" % table_name)
+
+    pandas_sql = PandasSQLAlchemy(con, meta=meta)
     table = pandas_sql.read_table(
         table_name, index_col=index_col, coerce_float=coerce_float,
         parse_dates=parse_dates, columns=columns)
@@ -380,6 +399,7 @@ def read_sql(sql, con, index_col=None, coerce_float=True, params=None,
             coerce_float=coerce_float, parse_dates=parse_dates)
 
     if pandas_sql.has_table(sql):
+        pandas_sql.meta.reflect(only=[sql])
         return pandas_sql.read_table(
             sql, index_col=index_col, coerce_float=coerce_float,
             parse_dates=parse_dates, columns=columns)
@@ -471,17 +491,9 @@ def pandasSQL_builder(con, flavor=None, meta=None, is_cursor=False):
     """
     # When support for DBAPI connections is removed,
     # is_cursor should not be necessary.
-    try:
-        import sqlalchemy
-
-        if isinstance(con, sqlalchemy.engine.Engine):
-            return PandasSQLAlchemy(con, meta=meta)
-        else:
-            if flavor == 'mysql':
-                warnings.warn(_MYSQL_WARNING, FutureWarning)
-            return PandasSQLLegacy(con, flavor, is_cursor=is_cursor)
-
-    except ImportError:
+    if _is_sqlalchemy_engine(con):
+        return PandasSQLAlchemy(con, meta=meta)
+    else:
         if flavor == 'mysql':
             warnings.warn(_MYSQL_WARNING, FutureWarning)
         return PandasSQLLegacy(con, flavor, is_cursor=is_cursor)
@@ -768,7 +780,6 @@ def __init__(self, engine, meta=None):
         if not meta:
             from sqlalchemy.schema import MetaData
             meta = MetaData(self.engine)
-            meta.reflect(self.engine)
 
         self.meta = meta
 
@@ -813,19 +824,16 @@ def tables(self):
         return self.meta.tables
 
     def has_table(self, name):
-        if self.meta.tables.get(name) is not None:
-            return True
-        else:
-            return False
+        return self.engine.has_table(name)
 
     def get_table(self, table_name):
         return self.meta.tables.get(table_name)
 
     def drop_table(self, table_name):
         if self.engine.has_table(table_name):
+            self.meta.reflect(only=[table_name])
             self.get_table(table_name).drop()
             self.meta.clear()
-            self.meta.reflect()
 
     def _create_sql_schema(self, frame, table_name):
         table = PandasSQLTable(table_name, self, frame=frame)

pandas/io/tests/test_sql.py

@@ -624,6 +624,22 @@ def test_read_sql_delegate(self):
         iris_frame2 = sql.read_sql('iris', self.conn)
         tm.assert_frame_equal(iris_frame1, iris_frame2)
 
+    def test_not_reflect_all_tables(self):
+        # create invalid table
+        qry = """CREATE TABLE invalid (x INTEGER, y UNKNOWN);"""
+        self.conn.execute(qry)
+        qry = """CREATE TABLE other_table (x INTEGER, y INTEGER);"""
+        self.conn.execute(qry)
+        
+        with warnings.catch_warnings(record=True) as w:
+            # Cause all warnings to always be triggered.
+            warnings.simplefilter("always")
+            # Trigger a warning.
+            sql.read_sql_table('other_table', self.conn)
+            sql.read_sql_query('SELECT * FROM other_table', self.conn)
+            # Verify some things
+            self.assertEqual(len(w), 0, "Warning triggered for other table")
+
 
 class TestSQLLegacyApi(_TestSQLApi):
     """
@@ -737,6 +753,8 @@ def setup_connect(self):
         try:
             self.conn = self.connect()
             self.pandasSQL = sql.PandasSQLAlchemy(self.conn)
+            # to test if connection can be made:
+            self.conn.connect()
         except sqlalchemy.exc.OperationalError:
             raise nose.SkipTest("Can't connect to {0} server".format(self.flavor))