[executors] feat: move convert_date and common macros instantiation to core

Author: AndreyMarkinPPC

libs/core/garf_core/query_editor.py

@@ -405,7 +405,10 @@ def macros(self) -> QueryParameters:
     """Returns macros with injected common parameters."""
     common_params = dict(self.common_params)
     if macros := self.args.macro:
-      common_params.update(macros)
+      converted_macros = {
+        key: convert_date(value) for key, value in macros.items()
+      }
+      common_params.update(converted_macros)
     return common_params
 
   def generate(self) -> BaseQueryElements:
@@ -564,3 +567,50 @@ def _is_invalid_field(field) -> bool:
   is_constant = _is_constant(field)
   has_operator = any(operator in field for operator in operators)
   return is_constant or has_operator
+
+
+def convert_date(date_string: str) -> str:
+  """Converts specific dates parameters to actual dates.
+
+  Returns:
+    Date string in YYYY-MM-DD format.
+
+  Raises:
+    GarfMacroError:
+     If dynamic lookback value (:YYYYMMDD-N) is incorrect.
+  """
+  if isinstance(date_string, list) or date_string.find(':Y') == -1:
+    return date_string
+  current_date = datetime.date.today()
+  base_date, *date_customizer = re.split('\\+|-', date_string)
+  if len(date_customizer) > 1:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+  if not date_customizer:
+    days_lookback = 0
+  else:
+    try:
+      days_lookback = int(date_customizer[0])
+    except ValueError as e:
+      raise GarfMacroError(
+        'Must provide numeric value for a number lookback period, '
+        'i.e. :YYYYMMDD-1'
+      ) from e
+  if base_date == ':YYYY':
+    new_date = datetime.datetime(current_date.year, 1, 1)
+    delta = relativedelta.relativedelta(years=days_lookback)
+  elif base_date == ':YYYYMM':
+    new_date = datetime.datetime(current_date.year, current_date.month, 1)
+    delta = relativedelta.relativedelta(months=days_lookback)
+  elif base_date == ':YYYYMMDD':
+    new_date = current_date
+    delta = relativedelta.relativedelta(days=days_lookback)
+  else:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+
+  if '-' in date_string:
+    return (new_date - delta).strftime('%Y-%m-%d')
+  return (new_date + delta).strftime('%Y-%m-%d')

libs/core/tests/unit/test_query_editor.py

@@ -17,6 +17,7 @@
 import datetime
 
 import pytest
+from dateutil.relativedelta import relativedelta
 from garf_core import query_editor
 
 
@@ -129,3 +130,67 @@ def test_generate_returns_builtin_query(self):
     )
 
     assert test_query_spec.query == expected_query_elements
+
+  def test_generate_returns_macros(self):
+    query = 'SELECT test FROM resource WHERE start_date = {start_date}'
+    test_query_spec = query_editor.QuerySpecification(
+      text=query,
+      title='test',
+      args=query_editor.GarfQueryParameters(
+        macro={'start_date': ':YYYYMMDD'},
+      ),
+    )
+    assert test_query_spec.macros.get(
+      'start_date'
+    ) == datetime.date.today().strftime('%Y-%m-%d')
+
+
+def test_convert_date_returns_correct_datestring():
+  current_date = datetime.datetime.today()
+  current_year = datetime.datetime(current_date.year, 1, 1)
+  current_month = datetime.datetime(current_date.year, current_date.month, 1)
+  last_year = current_year - relativedelta(years=1)
+  last_month = current_month - relativedelta(months=1)
+  yesterday = current_date - relativedelta(days=1)
+  tomorrow = current_date + relativedelta(days=1)
+  next_month = current_month + relativedelta(months=1)
+  next_year = current_year + relativedelta(years=1)
+
+  non_macro_date = '2022-01-01'
+  date_year = ':YYYY'
+  date_month = ':YYYYMM'
+  date_day = ':YYYYMMDD'
+  date_year_minus_one = ':YYYY-1'
+  date_month_minus_one = ':YYYYMM-1'
+  date_day_minus_one = ':YYYYMMDD-1'
+  date_day_plus_one = ':YYYYMMDD+1'
+  date_month_plus_one = ':YYYYMM+1'
+  date_year_plus_one = ':YYYY+1'
+
+  non_macro_date_converted = query_editor.convert_date(non_macro_date)
+  new_date_year = query_editor.convert_date(date_year)
+  new_date_month = query_editor.convert_date(date_month)
+  new_date_day = query_editor.convert_date(date_day)
+  new_date_year_minus_one = query_editor.convert_date(date_year_minus_one)
+  new_date_month_minus_one = query_editor.convert_date(date_month_minus_one)
+  new_date_day_minus_one = query_editor.convert_date(date_day_minus_one)
+  new_date_day_plus_one = query_editor.convert_date(date_day_plus_one)
+  new_date_month_plus_one = query_editor.convert_date(date_month_plus_one)
+  new_date_year_plus_one = query_editor.convert_date(date_year_plus_one)
+
+  assert non_macro_date_converted == non_macro_date
+  assert new_date_year == current_year.strftime('%Y-%m-%d')
+  assert new_date_month == current_month.strftime('%Y-%m-%d')
+  assert new_date_day == current_date.strftime('%Y-%m-%d')
+  assert new_date_year_minus_one == last_year.strftime('%Y-%m-%d')
+  assert new_date_month_minus_one == last_month.strftime('%Y-%m-%d')
+  assert new_date_day_minus_one == yesterday.strftime('%Y-%m-%d')
+  assert new_date_day_plus_one == tomorrow.strftime('%Y-%m-%d')
+  assert new_date_month_plus_one == next_month.strftime('%Y-%m-%d')
+  assert new_date_year_plus_one == next_year.strftime('%Y-%m-%d')
+
+
+@pytest.mark.parametrize('date', [':YYYYMMDD-N', ':YYYYMMDD-2-3', ':YYYMMDD'])
+def test_convert_date_raise_garf_macro_error(date):
+  with pytest.raises(query_editor.GarfMacroError):
+    query_editor.convert_date(date)

libs/executors/garf_executors/entrypoints/utils.py

@@ -15,18 +15,11 @@
 
 from __future__ import annotations
 
-import dataclasses
-import datetime
 import logging
-import os
 import sys
-from collections.abc import MutableSequence, Sequence
+from collections.abc import Sequence
 from typing import Any
 
-import smart_open
-import yaml
-from dateutil import relativedelta
-from garf_core import query_editor
 from rich import logging as rich_logging
 
 
@@ -96,43 +89,6 @@ class GarfParamsException(Exception):
   """Defines exception for incorrect parameters."""
 
 
-def convert_date(date_string: str) -> str:
-  """Converts specific dates parameters to actual dates.
-
-  Returns:
-      Date string in YYYY-MM-DD format.
-
-  Raises:
-      ValueError:
-          If dynamic lookback value (:YYYYMMDD-N) is incorrect.
-  """
-  if isinstance(date_string, list) or date_string.find(':YYYY') == -1:
-    return date_string
-  current_date = datetime.date.today()
-  date_object = date_string.split('-')
-  base_date = date_object[0]
-  if len(date_object) == 2:
-    try:
-      days_ago = int(date_object[1])
-    except ValueError as e:
-      raise ValueError(
-        'Must provide numeric value for a number lookback period, '
-        'i.e. :YYYYMMDD-1'
-      ) from e
-  else:
-    days_ago = 0
-  if base_date == ':YYYY':
-    new_date = datetime.datetime(current_date.year, 1, 1)
-    delta = relativedelta.relativedelta(years=days_ago)
-  elif base_date == ':YYYYMM':
-    new_date = datetime.datetime(current_date.year, current_date.month, 1)
-    delta = relativedelta.relativedelta(months=days_ago)
-  elif base_date == ':YYYYMMDD':
-    new_date = current_date
-    delta = relativedelta.relativedelta(days=days_ago)
-  return (new_date - delta).strftime('%Y-%m-%d')
-
-
 def init_logging(
   loglevel: str = 'INFO', logger_type: str = 'local', name: str = __name__
 ) -> logging.Logger:

libs/executors/tests/unit/entrypoints/test_utils.py

@@ -13,55 +13,10 @@
 # limitations under the License.
 from __future__ import annotations
 
-from datetime import datetime
-from unittest import mock
-
 import pytest
-import yaml
-from dateutil.relativedelta import relativedelta
-from garf_core.query_editor import CommonParametersMixin
 from garf_executors.entrypoints import utils
 
 
-def test_convert_date():
-  current_date = datetime.today()
-  current_year = datetime(current_date.year, 1, 1)
-  current_month = datetime(current_date.year, current_date.month, 1)
-  last_year = current_year - relativedelta(years=1)
-  last_month = current_month - relativedelta(months=1)
-  yesterday = current_date - relativedelta(days=1)
-
-  non_macro_date = '2022-01-01'
-  date_year = ':YYYY'
-  date_month = ':YYYYMM'
-  date_day = ':YYYYMMDD'
-  date_year_minus_one = ':YYYY-1'
-  date_month_minus_one = ':YYYYMM-1'
-  date_day_minus_one = ':YYYYMMDD-1'
-
-  non_macro_date_converted = utils.convert_date(non_macro_date)
-  new_date_year = utils.convert_date(date_year)
-  new_date_month = utils.convert_date(date_month)
-  new_date_day = utils.convert_date(date_day)
-  new_date_year_minus_one = utils.convert_date(date_year_minus_one)
-  new_date_month_minus_one = utils.convert_date(date_month_minus_one)
-  new_date_day_minus_one = utils.convert_date(date_day_minus_one)
-
-  assert non_macro_date_converted == non_macro_date
-  assert new_date_year == current_year.strftime('%Y-%m-%d')
-  assert new_date_month == current_month.strftime('%Y-%m-%d')
-  assert new_date_day == current_date.strftime('%Y-%m-%d')
-  assert new_date_year_minus_one == last_year.strftime('%Y-%m-%d')
-  assert new_date_month_minus_one == last_month.strftime('%Y-%m-%d')
-  assert new_date_day_minus_one == yesterday.strftime('%Y-%m-%d')
-
-
-def test_wrong_convert_date():
-  date_day = ':YYYYMMDD-N'
-  with pytest.raises(ValueError):
-    utils.convert_date(date_day)
-
-
 class TestParamsParser:
   @pytest.fixture
   def param_parser(self):

libs/executors/tests/unit/test_config.py

@@ -26,7 +26,7 @@ def test_from_file_returns_correct_context_from_data(self, tmp_path):
             'start_date': '2025-01-01',
           },
           'template': {
-            'cohorts': '1',
+            'cohorts': 1,
           },
         },
         'fetcher_parameters': {
@@ -53,7 +53,7 @@ def test_save_returns_correct_data(self, tmp_path):
             'start_date': '2025-01-01',
           },
           'template': {
-            'cohorts': '1',
+            'cohorts': 1,
           },
         },
         'fetcher_parameters': {

libs/executors/tests/unit/test_execution_context.py

@@ -33,7 +33,7 @@ def test_from_file_returns_correct_context_from_data(self, tmp_path):
           'start_date': '2025-01-01',
         },
         'template': {
-          'cohorts': [1, 2, 3, 4],
+          'cohorts': 1,
         },
       },
       'fetcher_parameters': {
@@ -58,7 +58,7 @@ def test_save_returns_correct_data(self, tmp_path):
           'start_date': '2025-01-01',
         },
         'template': {
-          'cohorts': [1, 2, 3, 4],
+          'cohorts': 1,
         },
       },
       'fetcher_parameters': {