feat(executors): workflow can process folder with queries

libs/executors/garf/executors/__init__.py

@@ -57,4 +57,4 @@ def setup_executor(
   'ApiExecutionContext',
 ]
 
-__version__ = '1.0.1'
+__version__ = '1.0.2'

libs/executors/garf/executors/entrypoints/cli.py

@@ -102,7 +102,18 @@ def main():
         for query in queries:
           if isinstance(query, garf.executors.workflow.QueryPath):
             query_path = wf_parent / pathlib.Path(query.path)
+            if not query_path.exists():
+              raise workflow.GarfWorkflowError(f'Query: {query_path} not found')
             batch[query.path] = reader_client.read(query_path)
+          elif isinstance(query, garf.executors.workflow.QueryFolder):
+            query_path = wf_parent / pathlib.Path(query.folder)
+            if not query_path.exists():
+              raise workflow.GarfWorkflowError(
+                f'Folder: {query_path} not found'
+              )
+            for p in query_path.rglob('*'):
+              if p.suffix == '.sql':
+                batch[p.stem] = reader_client.read(p)
           else:
             batch[query.query.title] = query.query.text
         query_executor.execute_batch(

libs/executors/garf/executors/workflow.py

@@ -27,6 +27,12 @@ class GarfWorkflowError(exceptions.GarfExecutorError):
   """Workflow specific exception."""
 
 
+class QueryFolder(pydantic.BaseModel):
+  """Path to folder with queries."""
+
+  folder: str
+
+
 class QueryPath(pydantic.BaseModel):
   """Path file with query."""
 
@@ -63,7 +69,7 @@ class ExecutionStep(ExecutionContext):
 
   fetcher: str | None = None
   alias: str | None = pydantic.Field(default=None, pattern=r'^[a-zA-Z0-9_]+$')
-  queries: list[QueryPath | QueryDefinition] | None = None
+  queries: list[QueryPath | QueryDefinition | QueryFolder] | None = None
 
   @property
   def context(self) -> ExecutionContext:

libs/executors/tests/unit/test_workflow.py

@@ -22,6 +22,7 @@ class TestWorkflow:
       {
         'fetcher': 'api',
         'queries': [
+          {'folder': 'queries'},
           {'path': 'example.sql'},
           {'query': {'text': 'SELECT 1', 'title': 'example2'}},
         ],