Merge pull request #129 from TyShkan/fix_recursive_folder_scan

fix: recursive folder scan

Author: pnadolny13

meltano.yml

@@ -22,10 +22,10 @@ plugins:
       add_metadata_columns: false
     settings:
       - name: files
-        description: Array of objects containing keys - `entity`, `file`, `keys`, `encoding` (Optional), `delimiter` (Optional), `doublequote` (Optional), `escapechar` (Optional), `quotechar` (Optional), `skipinitialspace` (Optional), `strict` (Optional)
+        description: Array of objects containing keys - `entity`, `path`, `keys`, `encoding` (Optional), `delimiter` (Optional), `doublequote` (Optional), `escapechar` (Optional), `quotechar` (Optional), `skipinitialspace` (Optional), `strict` (Optional)
         kind: array
       - name: csv_files_definition
-        description: "Project-relative path to JSON file holding array of objects with keys: `entity`, `file`, `keys`, and `encoding` (Optional)."
+        description: "Project-relative path to JSON file holding array of objects with keys: `entity`, `path`, `keys`, and `encoding` (Optional)."
         documentation: https://gitlab.com/meltano/tap-csv#run
         label: CSV Files Definition
         placeholder: Ex. files-def.json

tap_csv/client.py

@@ -50,6 +50,17 @@ def get_records(self, context: Optional[dict]) -> Iterable[dict]:
 
                 yield dict(zip(self.header, row))
 
+    def _get_recursive_file_paths(self, file_path: str) -> list:
+        file_paths = []
+
+        for dirpath, _, filenames in os.walk(file_path):
+            for filename in filenames:
+                file_path = os.path.join(dirpath, filename)
+                if self.is_valid_filename(file_path):
+                    file_paths.append(file_path)
+
+        return file_paths
+
     def get_file_paths(self) -> list:
         """Return a list of file paths to read.
 
@@ -67,10 +78,7 @@ def get_file_paths(self) -> list:
         file_paths = []
         if os.path.isdir(file_path):
             clean_file_path = os.path.normpath(file_path) + os.sep
-            for filename in os.listdir(clean_file_path):
-                file_path = clean_file_path + filename
-                if self.is_valid_filename(file_path):
-                    file_paths.append(file_path)
+            file_paths = self._get_recursive_file_paths(clean_file_path)
         else:
             if self.is_valid_filename(file_path):
                 file_paths.append(file_path)

tap_csv/tests/data/subfolder1/alphabet.csv

@@ -0,0 +1,2 @@
+col1,col2,col3
+a,b,c

tap_csv/tests/data/subfolder1/subfolder2/alphabet.csv

@@ -0,0 +1,2 @@
+col1,col2,col3
+d,e,f

tap_csv/tests/test_client.py

@@ -0,0 +1,30 @@
+"""Tests client methods."""
+
+import os
+
+from tap_csv.tap import CSVStream, TapCSV
+
+
+def test_get_file_paths_recursively():
+    """Test get file paths recursively."""
+    test_data_dir = os.path.dirname(os.path.abspath(__file__))
+
+    SAMPLE_CONFIG = {
+        "files": [
+            {
+                "entity": "test",
+                "path": f"{test_data_dir}/data/subfolder1/",
+                "keys": [],
+            }
+        ]
+    }
+
+    stream = CSVStream(
+        tap=TapCSV(config=SAMPLE_CONFIG, catalog={}, state={}),
+        name="test_recursive",
+        file_config=SAMPLE_CONFIG.get("files")[0],
+    )
+    assert stream.get_file_paths() == [
+        f"{test_data_dir}/data/subfolder1/alphabet.csv",
+        f"{test_data_dir}/data/subfolder1/subfolder2/alphabet.csv",
+    ]