Merge pull request #8668 from waveform80/deque-concat

BUG: fix concat to work with more iterables (GH8645)

Author: jreback

doc/source/whatsnew/v0.15.1.txt

@@ -75,6 +75,28 @@ API changes
 
      gr.apply(sum)
 
+- ``concat`` permits a wider variety of iterables of pandas objects to be
+  passed as the first parameter (:issue:`8645`):
+
+  .. ipython:: python
+
+     from collections import deque
+     df1 = pd.DataFrame([1, 2, 3])
+     df2 = pd.DataFrame([4, 5, 6])
+
+  previous behavior:
+
+  .. code-block:: python
+
+     In [7]: pd.concat(deque((df1, df2)))
+     TypeError: first argument must be a list-like of pandas objects, you passed an object of type "deque"
+
+  current behavior:
+
+  .. ipython:: python
+
+     pd.concat(deque((df1, df2)))
+
 .. _whatsnew_0151.enhancements:
 
 Enhancements

pandas/tools/merge.py

@@ -675,7 +675,7 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
 
     Parameters
     ----------
-    objs : list or dict of Series, DataFrame, or Panel objects
+    objs : a sequence or mapping of Series, DataFrame, or Panel objects
         If a dict is passed, the sorted keys will be used as the `keys`
         argument, unless it is passed, in which case the values will be
         selected (see below). Any None objects will be dropped silently unless
@@ -731,8 +731,8 @@ class _Concatenator(object):
     def __init__(self, objs, axis=0, join='outer', join_axes=None,
                  keys=None, levels=None, names=None,
                  ignore_index=False, verify_integrity=False, copy=True):
-        if not isinstance(objs, (list,tuple,types.GeneratorType,dict,TextFileReader)):
-            raise TypeError('first argument must be a list-like of pandas '
+        if isinstance(objs, (NDFrame, compat.string_types)):
+            raise TypeError('first argument must be an iterable of pandas '
                             'objects, you passed an object of type '
                             '"{0}"'.format(type(objs).__name__))
 

pandas/tools/tests/test_merge.py

@@ -2203,6 +2203,33 @@ def test_concat_series_axis1_same_names_ignore_index(self):
         result = concat([s1, s2], axis=1, ignore_index=True)
         self.assertTrue(np.array_equal(result.columns, [0, 1]))
 
+    def test_concat_iterables(self):
+        from collections import deque, Iterable
+
+        # GH8645 check concat works with tuples, list, generators, and weird
+        # stuff like deque and custom iterables
+        df1 = DataFrame([1, 2, 3])
+        df2 = DataFrame([4, 5, 6])
+        expected = DataFrame([1, 2, 3, 4, 5, 6])
+        assert_frame_equal(pd.concat((df1, df2), ignore_index=True), expected)
+        assert_frame_equal(pd.concat([df1, df2], ignore_index=True), expected)
+        assert_frame_equal(pd.concat((df for df in (df1, df2)), ignore_index=True), expected)
+        assert_frame_equal(pd.concat(deque((df1, df2)), ignore_index=True), expected)
+        class CustomIterator1(object):
+            def __len__(self):
+                return 2
+            def __getitem__(self, index):
+                try:
+                    return {0: df1, 1: df2}[index]
+                except KeyError:
+                    raise IndexError
+        assert_frame_equal(pd.concat(CustomIterator1(), ignore_index=True), expected)
+        class CustomIterator2(Iterable):
+            def __iter__(self):
+                yield df1
+                yield df2
+        assert_frame_equal(pd.concat(CustomIterator2(), ignore_index=True), expected)
+
     def test_concat_invalid(self):
 
         # trying to concat a ndframe with a non-ndframe