[executors] feat: add Config to store parameters for query execution

Config allows to capture all execution context parameters related to a particular source (writer, fetcher, macros).

* Add Config class with save and load methods; add tests for them
* Add methods to ExecutionContext to save and load particular execution context from a file
* Add support to garf cli to work with Config.
* Remove obsoleted BaseConfig class and its subclasses as well as methods operating on it; remove all related tests
* Move initialize_runtime_parameters helper function into `garf_core.query_editor` - now dynamic dates are converted to normal date strings during macro initialization.
*  Move convert_date to `garf_core.query_editor` and add support for look_forward dynamic dates.

Author: AndreyMarkinPPC

libs/core/garf_core/__init__.py

@@ -26,4 +26,4 @@
   'ApiReportFetcher',
 ]
 
-__version__ = '0.2.0'
+__version__ = '0.2.1'

libs/core/garf_core/query_editor.py

@@ -405,7 +405,10 @@ def macros(self) -> QueryParameters:
     """Returns macros with injected common parameters."""
     common_params = dict(self.common_params)
     if macros := self.args.macro:
-      common_params.update(macros)
+      converted_macros = {
+        key: convert_date(value) for key, value in macros.items()
+      }
+      common_params.update(converted_macros)
     return common_params
 
   def generate(self) -> BaseQueryElements:
@@ -564,3 +567,50 @@ def _is_invalid_field(field) -> bool:
   is_constant = _is_constant(field)
   has_operator = any(operator in field for operator in operators)
   return is_constant or has_operator
+
+
+def convert_date(date_string: str) -> str:
+  """Converts specific dates parameters to actual dates.
+
+  Returns:
+    Date string in YYYY-MM-DD format.
+
+  Raises:
+    GarfMacroError:
+     If dynamic lookback value (:YYYYMMDD-N) is incorrect.
+  """
+  if isinstance(date_string, list) or date_string.find(':Y') == -1:
+    return date_string
+  current_date = datetime.date.today()
+  base_date, *date_customizer = re.split('\\+|-', date_string)
+  if len(date_customizer) > 1:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+  if not date_customizer:
+    days_lookback = 0
+  else:
+    try:
+      days_lookback = int(date_customizer[0])
+    except ValueError as e:
+      raise GarfMacroError(
+        'Must provide numeric value for a number lookback period, '
+        'i.e. :YYYYMMDD-1'
+      ) from e
+  if base_date == ':YYYY':
+    new_date = datetime.datetime(current_date.year, 1, 1)
+    delta = relativedelta.relativedelta(years=days_lookback)
+  elif base_date == ':YYYYMM':
+    new_date = datetime.datetime(current_date.year, current_date.month, 1)
+    delta = relativedelta.relativedelta(months=days_lookback)
+  elif base_date == ':YYYYMMDD':
+    new_date = current_date
+    delta = relativedelta.relativedelta(days=days_lookback)
+  else:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+
+  if '-' in date_string:
+    return (new_date - delta).strftime('%Y-%m-%d')
+  return (new_date + delta).strftime('%Y-%m-%d')

libs/core/tests/unit/test_query_editor.py

@@ -17,6 +17,7 @@
 import datetime
 
 import pytest
+from dateutil.relativedelta import relativedelta
 from garf_core import query_editor
 
 
@@ -129,3 +130,67 @@ def test_generate_returns_builtin_query(self):
     )
 
     assert test_query_spec.query == expected_query_elements
+
+  def test_generate_returns_macros(self):
+    query = 'SELECT test FROM resource WHERE start_date = {start_date}'
+    test_query_spec = query_editor.QuerySpecification(
+      text=query,
+      title='test',
+      args=query_editor.GarfQueryParameters(
+        macro={'start_date': ':YYYYMMDD'},
+      ),
+    )
+    assert test_query_spec.macros.get(
+      'start_date'
+    ) == datetime.date.today().strftime('%Y-%m-%d')
+
+
+def test_convert_date_returns_correct_datestring():
+  current_date = datetime.datetime.today()
+  current_year = datetime.datetime(current_date.year, 1, 1)
+  current_month = datetime.datetime(current_date.year, current_date.month, 1)
+  last_year = current_year - relativedelta(years=1)
+  last_month = current_month - relativedelta(months=1)
+  yesterday = current_date - relativedelta(days=1)
+  tomorrow = current_date + relativedelta(days=1)
+  next_month = current_month + relativedelta(months=1)
+  next_year = current_year + relativedelta(years=1)
+
+  non_macro_date = '2022-01-01'
+  date_year = ':YYYY'
+  date_month = ':YYYYMM'
+  date_day = ':YYYYMMDD'
+  date_year_minus_one = ':YYYY-1'
+  date_month_minus_one = ':YYYYMM-1'
+  date_day_minus_one = ':YYYYMMDD-1'
+  date_day_plus_one = ':YYYYMMDD+1'
+  date_month_plus_one = ':YYYYMM+1'
+  date_year_plus_one = ':YYYY+1'
+
+  non_macro_date_converted = query_editor.convert_date(non_macro_date)
+  new_date_year = query_editor.convert_date(date_year)
+  new_date_month = query_editor.convert_date(date_month)
+  new_date_day = query_editor.convert_date(date_day)
+  new_date_year_minus_one = query_editor.convert_date(date_year_minus_one)
+  new_date_month_minus_one = query_editor.convert_date(date_month_minus_one)
+  new_date_day_minus_one = query_editor.convert_date(date_day_minus_one)
+  new_date_day_plus_one = query_editor.convert_date(date_day_plus_one)
+  new_date_month_plus_one = query_editor.convert_date(date_month_plus_one)
+  new_date_year_plus_one = query_editor.convert_date(date_year_plus_one)
+
+  assert non_macro_date_converted == non_macro_date
+  assert new_date_year == current_year.strftime('%Y-%m-%d')
+  assert new_date_month == current_month.strftime('%Y-%m-%d')
+  assert new_date_day == current_date.strftime('%Y-%m-%d')
+  assert new_date_year_minus_one == last_year.strftime('%Y-%m-%d')
+  assert new_date_month_minus_one == last_month.strftime('%Y-%m-%d')
+  assert new_date_day_minus_one == yesterday.strftime('%Y-%m-%d')
+  assert new_date_day_plus_one == tomorrow.strftime('%Y-%m-%d')
+  assert new_date_month_plus_one == next_month.strftime('%Y-%m-%d')
+  assert new_date_year_plus_one == next_year.strftime('%Y-%m-%d')
+
+
+@pytest.mark.parametrize('date', [':YYYYMMDD-N', ':YYYYMMDD-2-3', ':YYYMMDD'])
+def test_convert_date_raise_garf_macro_error(date):
+  with pytest.raises(query_editor.GarfMacroError):
+    query_editor.convert_date(date)

libs/executors/garf_executors/config.py

@@ -0,0 +1,51 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=C0330, g-bad-import-order, g-multiple-import
+
+"""Stores mapping between API aliases and their execution context."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+
+import pydantic
+import smart_open
+import yaml
+
+from garf_executors.execution_context import ExecutionContext
+
+
+class Config(pydantic.BaseModel):
+  """Stores necessary parameters for one or multiple API sources.
+
+  Attributes:
+    source: Mapping between API source alias and execution parameters.
+  """
+
+  sources: dict[str, ExecutionContext]
+
+  @classmethod
+  def from_file(cls, path: str | pathlib.Path | os.PathLike[str]) -> Config:
+    """Builds config from local or remote yaml file."""
+    with smart_open.open(path, 'r', encoding='utf-8') as f:
+      data = yaml.safe_load(f)
+    return Config(sources=data)
+
+  def save(self, path: str | pathlib.Path | os.PathLike[str]) -> str:
+    """Saves config to local or remote yaml file."""
+    with smart_open.open(path, 'w', encoding='utf-8') as f:
+      yaml.dump(self.model_dump().get('sources'), f, encoding='utf-8')
+    return f'Config is saved to {str(path)}'

libs/executors/garf_executors/entrypoints/cli.py

@@ -25,14 +25,14 @@
 from garf_io import reader
 
 import garf_executors
-from garf_executors import exceptions
+from garf_executors import config, exceptions
 from garf_executors.entrypoints import utils
 
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('query', nargs='*')
-  parser.add_argument('-c', '--config', dest='garf_config', default=None)
+  parser.add_argument('-c', '--config', dest='config', default=None)
   parser.add_argument('--source', dest='source', default=None)
   parser.add_argument('--output', dest='output', default='console')
   parser.add_argument('--input', dest='input', default='file')
@@ -64,34 +64,44 @@ def main():
     raise exceptions.GarfExecutorError(
       'Please provide one or more queries to run'
     )
-  config = utils.ConfigBuilder('garf').build(vars(args), kwargs)
-  logger.debug('config: %s', config)
-
-  if config.params:
-    config = utils.initialize_runtime_parameters(config)
-  logger.debug('initialized config: %s', config)
-
-  extra_parameters = utils.ParamsParser(['source']).parse(kwargs)
-  source_parameters = extra_parameters.get('source', {})
   reader_client = reader.create_reader(args.input)
-
-  context = garf_executors.api_executor.ApiExecutionContext(
-    query_parameters=config.params,
-    writer=args.output,
-    writer_parameters=config.writer_params,
-    fetcher_parameters=source_parameters,
-  )
-  query_executor = garf_executors.setup_executor(
-    args.source, context.fetcher_parameters
-  )
-  if args.parallel_queries:
-    logger.info('Running queries in parallel')
+  if config_file := args.config:
+    execution_config = config.Config.from_file(config_file)
+    if not (context := execution_config.sources.get(args.source)):
+      raise exceptions.GarfExecutorError(
+        f'No execution context found for source {args.source} in {config_file}'
+      )
+    query_executor = garf_executors.setup_executor(
+      args.source, context.fetcher_parameters
+    )
     batch = {query: reader_client.read(query) for query in args.query}
     query_executor.execute_batch(batch, context, args.parallel_queries)
   else:
-    logger.info('Running queries sequentially')
-    for query in args.query:
-      query_executor.execute(reader_client.read(query), query, context)
+    extra_parameters = utils.ParamsParser(
+      ['source', args.output, 'macro', 'template']
+    ).parse(kwargs)
+    source_parameters = extra_parameters.get('source', {})
+
+    context = garf_executors.api_executor.ApiExecutionContext(
+      query_parameters={
+        'macro': extra_parameters.get('macro'),
+        'template': extra_parameters.get('template'),
+      },
+      writer=args.output,
+      writer_parameters=extra_parameters.get(args.output),
+      fetcher_parameters=source_parameters,
+    )
+    query_executor = garf_executors.setup_executor(
+      args.source, context.fetcher_parameters
+    )
+    if args.parallel_queries:
+      logger.info('Running queries in parallel')
+      batch = {query: reader_client.read(query) for query in args.query}
+      query_executor.execute_batch(batch, context, args.parallel_queries)
+    else:
+      logger.info('Running queries sequentially')
+      for query in args.query:
+        query_executor.execute(reader_client.read(query), query, context)
 
 
 if __name__ == '__main__':