Make walk_files traverse in alphabetically breadth-first order.

Author: mthrok

test/datasets/libritts_test.py

@@ -49,16 +49,13 @@ def setUpClass(cls):
 
     def test_libritts(self):
         dataset = LIBRITTS(self.root_dir)
-        samples = list(dataset)
-        samples.sort(key=lambda s: s[4])
-
         for i, (waveform,
                 sample_rate,
                 original_text,
                 normalized_text,
                 speaker_id,
                 chapter_id,
-                utterance_id) in enumerate(samples):
+                utterance_id) in enumerate(dataset):
 
             expected_ids = self.utterance_ids[i]
             expected_data = self.data[i]

test/datasets/utils_test.py

@@ -0,0 +1,44 @@
+import os
+from pathlib import Path
+
+from torchaudio.datasets import utils as dataset_utils
+
+from ..common_utils import (
+    TempDirMixin,
+    TorchaudioTestCase,
+)
+
+
+class TestWalkFiles(TempDirMixin, TorchaudioTestCase):
+    root = None
+    expected = None
+
+    def _add_file(self, *parts):
+        path = self.get_temp_path(*parts)
+        self.expected.append(path)
+        Path(path).touch()
+
+    def setUp(self):
+        self.root = self.get_temp_path()
+        self.expected = []
+
+        # level 1
+        for filename in ['a.txt', 'b.txt', 'c.txt']:
+            self._add_file(filename)
+
+        # level 2
+        for dir1 in ['d1', 'd2', 'd3']:
+            for filename in ['d.txt', 'e.txt', 'f.txt']:
+                self._add_file(dir1, filename)
+            # level 3
+            for dir2 in ['d1', 'd2', 'd3']:
+                for filename in ['g.txt', 'h.txt', 'i.txt']:
+                    self._add_file(dir1, dir2, filename)
+
+        print('\n'.join(self.expected))
+
+    def test_walk_files(self):
+        """walk_files should traverse files in alphabetical order"""
+        for i, path in enumerate(dataset_utils.walk_files(self.root, '.txt', prefix=True)):
+            found = os.path.join(self.root, path)
+            assert found == self.expected[i]

test/datasets/yesno_test.py

@@ -38,9 +38,7 @@ def setUpClass(cls):
 
     def test_yesno(self):
         dataset = yesno.YESNO(self.root_dir)
-        samples = list(dataset)
-        samples.sort(key=lambda s: s[2])
-        for i, (waveform, sample_rate, label) in enumerate(samples):
+        for i, (waveform, sample_rate, label) in enumerate(dataset):
             expected_label = self.labels[i]
             expected_data = self.data[i]
             self.assertEqual(expected_data, waveform, atol=5e-5, rtol=1e-8)

torchaudio/datasets/utils.py

@@ -264,7 +264,13 @@ def walk_files(root: str,
 
     root = os.path.expanduser(root)
 
-    for dirpath, _, files in os.walk(root):
+    for dirpath, dirs, files in os.walk(root):
+        dirs.sort()
+        # `dirs` is the list used in os.walk function and by sorting it in-place here, we change the
+        # behavior of os.walk to traverse sub directory alphabetically
+        # see also
+        # https://stackoverflow.com/questions/6670029/can-i-force-python3s-os-walk-to-visit-directories-in-alphabetical-order-how#comment71993866_6670926
+        files.sort()
         for f in files:
             if f.endswith(suffix):