implement support for virtual columns in query_editor and parser

libs/core/garf_core/__init__.py

@@ -26,4 +26,4 @@
   'ApiReportFetcher',
 ]
 
-__version__ = '0.1.5'
+__version__ = '0.2.0'

libs/core/garf_core/api_clients.py

@@ -31,12 +31,13 @@
 from garf_core import exceptions, query_editor
 
 ApiRowElement: TypeAlias = Union[int, float, str, bool, list, dict, None]
+ApiResponseRow: TypeAlias = dict[str, ApiRowElement]
 
 
 class GarfApiResponse(pydantic.BaseModel):
   """Base class for specifying response."""
 
-  results: list[ApiRowElement]
+  results: list[ApiResponseRow]
 
 
 class GarfApiError(exceptions.GarfError):

libs/core/garf_core/parsers.py

@@ -16,16 +16,23 @@
 from __future__ import annotations
 
 import abc
+import ast
 import contextlib
 import functools
 import operator
 from collections.abc import Mapping, MutableSequence
 from typing import Any
 
-from typing_extensions import override
-
 from garf_core import api_clients, exceptions, query_editor
 
+VALID_VIRTUAL_COLUMN_OPERATORS = (
+  ast.BinOp,
+  ast.UnaryOp,
+  ast.operator,
+  ast.Constant,
+  ast.Expression,
+)
+
 
 class BaseParser(abc.ABC):
   """An interface for all parsers to implement."""
@@ -48,52 +55,111 @@ def parse_response(
       results.append(self.parse_row(result))
     return results
 
-  @abc.abstractmethod
-  def parse_row(self, row):
-    """Parses single row from response."""
-
-
-class ListParser(BaseParser):
-  """Returns API results as is."""
+  def _evalute_virtual_column(
+    self,
+    fields: list[str],
+    virtual_column_values: dict[str, Any],
+    substitute_expression: str,
+  ) -> api_clients.ApiRowElement:
+    virtual_column_replacements = {
+      field.replace('.', '_'): value
+      for field, value in zip(fields, virtual_column_values)
+    }
+    virtual_column_expression = substitute_expression.format(
+      **virtual_column_replacements
+    )
+    try:
+      tree = ast.parse(virtual_column_expression, mode='eval')
+      valid = all(
+        isinstance(node, VALID_VIRTUAL_COLUMN_OPERATORS)
+        for node in ast.walk(tree)
+      )
+      if valid:
+        return eval(
+          compile(tree, filename='', mode='eval'), {'__builtins__': None}
+        )
+    except ZeroDivisionError:
+      return 0
+    return None
+
+  def process_virtual_column(
+    self,
+    row: api_clients.ApiResponseRow,
+    virtual_column: query_editor.VirtualColumn,
+  ) -> api_clients.ApiRowElement:
+    if virtual_column.type == 'built-in':
+      return virtual_column.value
+    virtual_column_values = [
+      self.parse_row_element(row, field) for field in virtual_column.fields
+    ]
+    try:
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except TypeError:
+      virtual_column_values = [
+        f"'{self.parse_row_element(row, field)}'"
+        for field in virtual_column.fields
+      ]
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except SyntaxError:
+      return virtual_column.value
+    return result
 
-  @override
   def parse_row(
     self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    return row
+    row: api_clients.ApiResponseRow,
+  ) -> list[api_clients.ApiRowElement]:
+    """Parses single row from response."""
+    results = []
+    fields = self.query_spec.fields
+    index = 0
+    for column in self.query_spec.column_names:
+      if virtual_column := self.query_spec.virtual_columns.get(column):
+        result = self.process_virtual_column(row, virtual_column)
+      else:
+        result = self.parse_row_element(row, fields[index])
+        index = index + 1
+      results.append(result)
+    return results
+
+  @abc.abstractmethod
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
 
 
 class DictParser(BaseParser):
   """Extracts nested dict elements."""
 
-  @override
-  def parse_row(
-    self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
     if not isinstance(row, Mapping):
       raise GarfParserError
-    result = []
-    for field in self.query_spec.fields:
-      result.append(self.get_nested_field(row, field))
-    return result
-
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
-    """Returns nested fields from a dictionary."""
-    if result := dictionary.get(key):
+    if result := row.get(key):
       return result
     key = key.split('.')
     try:
-      return functools.reduce(operator.getitem, key, dictionary)
+      return functools.reduce(operator.getitem, key, row)
     except (TypeError, KeyError):
       return None
 
 
 class NumericConverterDictParser(DictParser):
   """Extracts nested dict elements with numerical conversions."""
 
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
     """Extract nested field with int/float conversion."""
 
     def convert_field(value):
@@ -102,12 +168,12 @@ def convert_field(value):
           return type_(value)
       return value
 
-    if result := dictionary.get(key):
+    if result := row.get(key):
       return convert_field(result)
 
     key = key.split('.')
     try:
-      field = functools.reduce(operator.getitem, key, dictionary)
+      field = functools.reduce(operator.getitem, key, row)
       if isinstance(field, MutableSequence) or field in (True, False):
         return field
       return convert_field(field)

libs/core/garf_core/query_editor.py

@@ -125,6 +125,8 @@ def _extract_pointer(cls, line_elements: str) -> list[str]:
 
   @classmethod
   def _extract_nested_resource(cls, line_elements: str) -> list[str]:
+    if '://' in line_elements:
+      return []
     return re.split(':', line_elements)
 
 
@@ -160,22 +162,29 @@ def from_raw(cls, field: str, macros: QueryParameters) -> VirtualColumn:
       return VirtualColumn(type='built-in', value=field)
 
     operators = ('/', r'\*', r'\+', ' - ')
-    if len(expressions := re.split('|'.join(operators), field)) > 1:
+    if '://' in field:
+      expressions = re.split(r'\+', field)
+    else:
+      expressions = re.split('|'.join(operators), field)
+    if len(expressions) > 1:
       virtual_column_fields = []
       substitute_expression = field
       for expression in expressions:
         element = expression.strip()
-        if True:
-          # if self._is_valid_field(element):
+        if not _is_constant(element):
           virtual_column_fields.append(element)
           substitute_expression = substitute_expression.replace(
             element, f'{{{element}}}'
           )
+      pattern = r'\{([^}]*)\}'
+      substitute_expression = re.sub(
+        pattern, lambda m: m.group(0).replace('.', '_'), substitute_expression
+      )
       return VirtualColumn(
         type='expression',
         value=field.format(**macros) if macros else field,
         fields=virtual_column_fields,
-        substitute_expression=substitute_expression.replace('.', '_'),
+        substitute_expression=substitute_expression,
       )
     if not _is_quoted_string(field):
       raise GarfFieldError(f"Incorrect field '{field}'.")
@@ -211,7 +220,11 @@ def from_query_line(
     field, *alias = re.split(' [Aa][Ss] ', line)
     processed_field = ProcessedField.from_raw(field)
     field = processed_field.field
-    virtual_column = None if field else VirtualColumn.from_raw(field, macros)
+    virtual_column = (
+      VirtualColumn.from_raw(field, macros)
+      if _is_invalid_field(field)
+      else None
+    )
     if alias and processed_field.customizer_type:
       customizer = {
         'type': processed_field.customizer_type,
@@ -435,10 +448,10 @@ def remove_comments(self) -> Self:
       if re.match('/\\*', line) or multiline_comment:
         multiline_comment = True
         continue
-      if re.match('^(#|--|//)', line):
+      if re.match('^(#|--|//) ', line):
         continue
       cleaned_query_line = re.sub(
-        ';$', '', re.sub('(--|//).*$', '', line).strip()
+        ';$', '', re.sub('(--|//) .*$', '', line).strip()
       )
       result.append(cleaned_query_line)
     self.query.text = ' '.join(result)
@@ -506,6 +519,10 @@ def extract_virtual_columns(self) -> Self:
       line_elements = ExtractedLineElements.from_query_line(line)
       if virtual_column := line_elements.virtual_column:
         self.query.virtual_columns[line_elements.alias] = virtual_column
+        if fields := virtual_column.fields:
+          for field in fields:
+            if field not in self.query.fields:
+              self.query.fields.append(field)
     return self
 
   def extract_customizers(self) -> Self:
@@ -533,3 +550,17 @@ def _is_quoted_string(field_name: str) -> bool:
   return (field_name.startswith("'") and field_name.endswith("'")) or (
     field_name.startswith('"') and field_name.endswith('"')
   )
+
+
+def _is_constant(element) -> bool:
+  with contextlib.suppress(ValueError):
+    float(element)
+    return True
+  return _is_quoted_string(element)
+
+
+def _is_invalid_field(field) -> bool:
+  operators = ('/', '*', '+', ' - ')
+  is_constant = _is_constant(field)
+  has_operator = any(operator in field for operator in operators)
+  return is_constant or has_operator

libs/core/garf_core/report_fetcher.py

@@ -54,16 +54,18 @@ class ApiReportFetcher:
   """Class responsible for getting data from report API.
 
   Attributes:
-      api_client: a client used for connecting to API.
-      parser: Type of parser to convert API response.
+      api_client: Client used for connecting to API.
+      parser: Class of parser to convert API response.
       query_specification_builder: Class to perform query parsing.
+      builtin_queries:
+        Mapping between query name and function for generating GarfReport.
   """
 
   def __init__(
     self,
-    api_client: api_clients.BaseApiClient,
-    parser: parsers.BaseParser = parsers.DictParser,
-    query_specification_builder: query_editor.QuerySpecification = (
+    api_client: api_clients.BaseClient,
+    parser: type[parsers.BaseParser] = parsers.DictParser,
+    query_specification_builder: type[query_editor.QuerySpecification] = (
       query_editor.QuerySpecification
     ),
     builtin_queries: dict[str, Callable[[ApiReportFetcher], report.GarfReport]]

libs/core/tests/unit/test_parsers.py

@@ -56,14 +56,6 @@ def test_parse_response_returns_empty_list_on_missing_results(
 
     assert parsed_row == expected_row
 
-  def test_parse_response_raises_garf_parse_error_on_incorrect_items(
-    self, test_parser
-  ):
-    test_response = api_clients.GarfApiResponse(results=[[1, 2]])
-
-    with pytest.raises(parsers.GarfParserError):
-      test_parser.parse_response(test_response)
-
   def test_parse_response_returns_none_for_missing_field(self):
     test_specification = query_editor.QuerySpecification(
       'SELECT test_column_1, missing_column.field FROM test'
@@ -94,16 +86,3 @@ def test_parse_row_returns_converted_numeric_values(self, test_parser):
     expected_row = [1]
 
     assert parsed_row == expected_row
-
-
-class TestListParser:
-  @pytest.fixture
-  def test_parser(self):
-    return parsers.ListParser(test_specification)
-
-  def test_parse_row_returns_converted_numeric_values(self, test_parser):
-    test_row = {'test_column_1': '1', 'test_column_2': 2}
-
-    parsed_row = test_parser.parse_row(test_row)
-
-    assert parsed_row == test_row