diff --git a/libs/executors/garf/executors/__init__.py b/libs/executors/garf/executors/__init__.py index 94fbb81..14eafa1 100644 --- a/libs/executors/garf/executors/__init__.py +++ b/libs/executors/garf/executors/__init__.py @@ -57,4 +57,4 @@ def setup_executor( 'ApiExecutionContext', ] -__version__ = '1.0.1' +__version__ = '1.0.2' diff --git a/libs/executors/garf/executors/entrypoints/cli.py b/libs/executors/garf/executors/entrypoints/cli.py index 2c96783..3536bc6 100644 --- a/libs/executors/garf/executors/entrypoints/cli.py +++ b/libs/executors/garf/executors/entrypoints/cli.py @@ -102,7 +102,18 @@ def main(): for query in queries: if isinstance(query, garf.executors.workflow.QueryPath): query_path = wf_parent / pathlib.Path(query.path) + if not query_path.exists(): + raise workflow.GarfWorkflowError(f'Query: {query_path} not found') batch[query.path] = reader_client.read(query_path) + elif isinstance(query, garf.executors.workflow.QueryFolder): + query_path = wf_parent / pathlib.Path(query.folder) + if not query_path.exists(): + raise workflow.GarfWorkflowError( + f'Folder: {query_path} not found' + ) + for p in query_path.rglob('*'): + if p.suffix == '.sql': + batch[p.stem] = reader_client.read(p) else: batch[query.query.title] = query.query.text query_executor.execute_batch( diff --git a/libs/executors/garf/executors/workflow.py b/libs/executors/garf/executors/workflow.py index 1e28877..cdec82e 100644 --- a/libs/executors/garf/executors/workflow.py +++ b/libs/executors/garf/executors/workflow.py @@ -27,6 +27,12 @@ class GarfWorkflowError(exceptions.GarfExecutorError): """Workflow specific exception.""" +class QueryFolder(pydantic.BaseModel): + """Path to folder with queries.""" + + folder: str + + class QueryPath(pydantic.BaseModel): """Path file with query.""" @@ -63,7 +69,7 @@ class ExecutionStep(ExecutionContext): fetcher: str | None = None alias: str | None = pydantic.Field(default=None, pattern=r'^[a-zA-Z0-9_]+$') - queries: list[QueryPath | QueryDefinition] | None = None + queries: list[QueryPath | QueryDefinition | QueryFolder] | None = None @property def context(self) -> ExecutionContext: diff --git a/libs/executors/tests/unit/test_workflow.py b/libs/executors/tests/unit/test_workflow.py index 5a7356a..bc735b8 100644 --- a/libs/executors/tests/unit/test_workflow.py +++ b/libs/executors/tests/unit/test_workflow.py @@ -22,6 +22,7 @@ class TestWorkflow: { 'fetcher': 'api', 'queries': [ + {'folder': 'queries'}, {'path': 'example.sql'}, {'query': {'text': 'SELECT 1', 'title': 'example2'}}, ],