[executors] feat: add google cloud logging option

Author: AndreyMarkinPPC

libs/core/garf_core/query_editor.py

@@ -28,6 +28,8 @@
 
 from garf_core import query_parser
 
+logger = logging.getLogger(__name__)
+
 QueryParameters: TypeAlias = dict[str, Union[str, float, int, list]]
 
 
@@ -126,22 +128,22 @@ class TemplateProcessorMixin:
   def replace_params_template(
     self, query_text: str, params: GarfQueryParameters | None = None
   ) -> str:
-    logging.debug('Original query text:\n%s', query_text)
+    logger.debug('Original query text:\n%s', query_text)
     if params:
       if templates := params.template:
         query_templates = {
           name: value for name, value in templates.items() if name in query_text
         }
         if query_templates:
           query_text = self.expand_jinja(query_text, query_templates)
-          logging.debug('Query text after jinja expansion:\n%s', query_text)
+          logger.debug('Query text after jinja expansion:\n%s', query_text)
         else:
           query_text = self.expand_jinja(query_text, {})
       else:
         query_text = self.expand_jinja(query_text, {})
       if macros := params.macro:
         query_text = query_text.format(**macros)
-        logging.debug('Query text after macro substitution:\n%s', query_text)
+        logger.debug('Query text after macro substitution:\n%s', query_text)
     else:
       query_text = self.expand_jinja(query_text, {})
     return query_text

libs/executors/garf_executors/entrypoints/cli.py

@@ -20,6 +20,7 @@
 from __future__ import annotations
 
 import argparse
+import logging
 import sys
 
 from garf_io import reader
@@ -102,6 +103,7 @@ def main():
       logger.info('Running queries sequentially')
       for query in args.query:
         query_executor.execute(reader_client.read(query), query, context)
+  logging.shutdown()
 
 
 if __name__ == '__main__':

libs/executors/garf_executors/entrypoints/utils.py

@@ -93,13 +93,15 @@ class GarfParamsException(Exception):
 class LoggerEnum(str, enum.Enum):
   local = 'local'
   rich = 'rich'
+  gcloud = 'gcloud'
 
 
 def init_logging(
   loglevel: str = 'INFO',
   logger_type: str | LoggerEnum = 'local',
   name: str = __name__,
 ) -> logging.Logger:
+  loglevel = getattr(logging, loglevel)
   if logger_type == 'rich':
     logging.basicConfig(
       format='%(message)s',
@@ -109,6 +111,17 @@ def init_logging(
         rich_logging.RichHandler(rich_tracebacks=True),
       ],
     )
+  elif logger_type == 'gcloud':
+    import google.cloud.logging as glogging
+
+    client = glogging.Client()
+    handler = glogging.handlers.CloudLoggingHandler(client, name='garf')
+    handler.close()
+    glogging.handlers.setup_logging(handler, log_level=loglevel)
+    logging.basicConfig(
+      level=loglevel,
+      handlers=[handler],
+    )
   else:
     logging.basicConfig(
       format='[%(asctime)s][%(name)s][%(levelname)s] %(message)s',

libs/executors/pyproject.toml

@@ -40,6 +40,7 @@ version = {attr = "garf_executors.__version__"}
 bq=[
   "garf-io[bq]",
   "pandas",
+  "google-cloud-logging",
 ]
 sql=[
   "garf-io[sqlalchemy]",

libs/exporters/garf_exporter/exporter.py

@@ -165,7 +165,7 @@ def _define_metrics(
     if virtual_columns := query_specification.virtual_columns:
       for column, field in virtual_columns.items():
         metrics[column] = self._define_gauge(column, suffix, labels)
-    logging.debug('metrics: %s', metrics)
+    logger.debug('metrics: %s', metrics)
     return metrics
 
   def _define_labels(

libs/io/garf_io/writers/bigquery_writer.py

@@ -38,6 +38,8 @@
 from garf_io import exceptions, formatter
 from garf_io.writers import abs_writer
 
+logger = logging.getLogger(__name__)
+
 
 class BigQueryWriterError(exceptions.GarfIoError):
   """BigQueryWriter specific errors."""
@@ -136,13 +138,13 @@ def write(self, report: garf_report.GarfReport, destination: str) -> str:
     else:
       df = report.to_pandas()
     df = df.replace({np.nan: None})
-    logging.debug('Writing %d rows of data to %s', len(df), destination)
+    logger.debug('Writing %d rows of data to %s', len(df), destination)
     job = self.client.load_table_from_dataframe(
       dataframe=df, destination=table, job_config=job_config
     )
     try:
       job.result()
-      logging.debug('Writing to %s is completed', destination)
+      logger.debug('Writing to %s is completed', destination)
     except google_cloud_exceptions.BadRequest as e:
       raise ValueError(f'Unable to save data to BigQuery! {str(e)}') from e
     return f'[BigQuery] - at {self.dataset_id}.{destination}'

libs/io/garf_io/writers/csv_writer.py

@@ -30,6 +30,8 @@
 from garf_io import formatter
 from garf_io.writers import file_writer
 
+logger = logging.getLogger(__name__)
+
 
 class CsvWriter(file_writer.FileWriter):
   """Writes Garf Report to CSV.
@@ -83,7 +85,7 @@ def write(self, report: garf_report.GarfReport, destination: str) -> str:
     report = self.format_for_write(report)
     destination = formatter.format_extension(destination, new_extension='.csv')
     self.create_dir()
-    logging.debug('Writing %d rows of data to %s', len(report), destination)
+    logger.debug('Writing %d rows of data to %s', len(report), destination)
     output_path = pathlib.Path(self.destination_folder) / destination
     with smart_open.open(
       output_path,
@@ -98,5 +100,5 @@ def write(self, report: garf_report.GarfReport, destination: str) -> str:
       )
       writer.writerow(report.column_names)
       writer.writerows(report.results)
-    logging.debug('Writing to %s is completed', output_path)
+    logger.debug('Writing to %s is completed', output_path)
     return f'[CSV] - at {output_path}'

libs/io/garf_io/writers/json_writer.py

@@ -26,6 +26,8 @@
 from garf_io import formatter
 from garf_io.writers import file_writer
 
+logger = logging.getLogger(__name__)
+
 
 class JsonWriter(file_writer.FileWriter):
   """Writes Garf Report to JSON.
@@ -68,9 +70,9 @@ def write(self, report: garf_report.GarfReport, destination: str) -> str:
       destination, new_extension=file_extension
     )
     self.create_dir()
-    logging.debug('Writing %d rows of data to %s', len(report), destination)
+    logger.debug('Writing %d rows of data to %s', len(report), destination)
     output_path = pathlib.Path(self.destination_folder) / destination
     with smart_open.open(output_path, 'w', encoding='utf-8') as f:
       f.write(report.to_json(output=self.format))
-    logging.debug('Writing to %s is completed', output_path)
+    logger.debug('Writing to %s is completed', output_path)
     return f'[JSON] - at {output_path}'

libs/io/garf_io/writers/sheets_writer.py

@@ -35,6 +35,8 @@
 from garf_io import exceptions, formatter
 from garf_io.writers.abs_writer import AbsWriter
 
+logger = logging.getLogger(__name__)
+
 
 class SheetWriterError(exceptions.GarfIoError):
   """SheetWriterError specific errors."""
@@ -94,7 +96,7 @@ def write(
       sheet.append_rows(report.results, value_input_option='RAW')
 
     success_msg = f'Report is saved to {sheet.url}'
-    logging.info(success_msg)
+    logger.info(success_msg)
     if self.share_with:
       self.spreadsheet.share(self.share_with, perm_type='user', role='writer')
     return success_msg

libs/io/garf_io/writers/sqldb_writer.py

@@ -31,6 +31,8 @@
 from garf_io import formatter
 from garf_io.writers import abs_writer
 
+logger = logging.getLogger(__name__)
+
 
 class SqlAlchemyWriter(abs_writer.AbsWriter):
   """Handles writing GarfReports data to databases supported by SqlAlchemy.
@@ -77,7 +79,7 @@ def write(self, report: garf_report.GarfReport, destination: str) -> None:
       ).head(0)
     else:
       df = report.to_pandas()
-    logging.debug('Writing %d rows of data to %s', len(df), destination)
+    logger.debug('Writing %d rows of data to %s', len(df), destination)
     write_params = {
       'name': destination,
       'con': self.engine,
@@ -87,7 +89,7 @@ def write(self, report: garf_report.GarfReport, destination: str) -> None:
     if dtypes:
       write_params.update({'dtype': dtypes})
     df.to_sql(**write_params)
-    logging.debug('Writing to %s is completed', destination)
+    logger.debug('Writing to %s is completed', destination)
 
   @property
   def engine(self) -> sqlalchemy.engine.Engine: