add back the path check, add valid/unvalid path check

Author: zhuoran-Cheng16

src/acquisition/covidcast/csv_importer.py

@@ -88,7 +88,7 @@ def find_issue_specific_csv_files(scan_dir, glob=glob):
     logger = get_structured_logger('find_issue_specific_csv_files')
     for path in sorted(glob.glob(os.path.join(scan_dir, '*'))):
       issuedir_match = CsvImporter.PATTERN_ISSUE_DIR.match(path.lower())
-      if issuedir_match:
+      if issuedir_match and os.path.isdir(path):
         issue_date_value = int(issuedir_match.group(2))
         issue_date = CsvImporter.is_sane_day(issue_date_value)
         if issue_date:

tests/acquisition/covidcast/test_csv_importer.py

@@ -6,7 +6,7 @@
 from datetime import date
 import math
 import numpy as np
-
+import os
 # third party
 import pandas
 import epiweeks as epi
@@ -44,23 +44,33 @@ def test_is_sane_week(self):
 
   def test_find_issue_specific_csv_files(self):
       """Recursively explore and find issue specific CSV files."""
-      path_prefix='prefix/to/the/data/'
-      #valid day path
-      issue_path='issue_20200408'
-      glob_issue_path = path_prefix + issue_path
-      glob_file_path = path_prefix + issue_path + '/ght/20200408_state_rawsearch.csv'
+      # valid path
+      path_prefix='prefix/to/the/data/issue_20200408'
+      os.makedirs(path_prefix, exist_ok=True)
+      self.assertTrue(os.path.isdir(path_prefix))
+
+      issue_path=path_prefix+'ght/20200408_state_rawsearch.csv'
       mock_glob = MagicMock()
-      mock_glob.glob.side_effect = ([glob_issue_path], [glob_file_path])
+      mock_glob.glob.side_effect = ([path_prefix], [issue_path])
 
-      #check if the day is a valid issue day.
-      issuedir_match = CsvImporter.PATTERN_ISSUE_DIR.match(glob_issue_path.lower())
+      #check if the day is a valid day.
+      issuedir_match= CsvImporter.PATTERN_ISSUE_DIR.match(path_prefix.lower())
       issue_date_value = int(issuedir_match.group(2))
-  
       self.assertTrue(CsvImporter.is_sane_day(issue_date_value))
-
+      
       found = set(CsvImporter.find_issue_specific_csv_files(path_prefix, glob=mock_glob))
       self.assertTrue(len(found)>0)
 
+      # unvalid path: 
+      path_prefix_invalid='invalid/prefix/to/the/data/issue_20200408'
+      self.assertFalse(os.path.isdir(path_prefix_invalid))
+      issue_path_invalid=path_prefix_invalid+'ght/20200408_state_rawsearch.csv'
+      mock_glob_invalid = MagicMock()
+      mock_glob_invalid.glob.side_effect = ([path_prefix_invalid], [issue_path_invalid])
+      found = set(CsvImporter.find_issue_specific_csv_files(path_prefix_invalid, glob=mock_glob_invalid))
+      self.assertFalse(len(found)>0)
+
+
   def test_find_csv_files(self):
     """Recursively explore and find CSV files."""