Dv/skruk/docstring cleanup

.pylintrc

@@ -1,11 +1,11 @@
 [MESSAGES CONTROL]
-disable=C0111,C0415,C0301,W0621,W0611,E0401,C0411,R0914,R0902,R0903,R1737,R0912,W0107,C0103,W1203,R0915,W0511,C0114,C0115,C0116,W0212,E0611,W0613,R1702,R1718,W0150
+disable=C0415,C0301,W0621,W0611,E0401,C0411,R0914,R0902,R0903,R1737,R0912,W0107,C0103,W1203,R0915,W0511,W0212,E0611,W0613,R1702,R1718,W0150
 
 [FORMAT]
 max-line-length=140
 
 [MASTER]
-ignore=.git,__pycache__,build,dist,.venv
+ignore=.git,__pycache__,build,dist,.venv,test
 
 [DESIGN]
 max-args=10

.vscode/settings.json

@@ -44,6 +44,7 @@
         "DATEADD",
         "deadsnakes",
         "dedup",
+        "descr",
         "devel",
         "docstrings",
         "doesn't",
@@ -66,6 +67,7 @@
         "Kamatchi",
         "kvlist",
         "LDATA",
+        "levelname",
         "libcairo",
         "libgdk",
         "libpango",
@@ -82,16 +84,19 @@
         "multitenancy",
         "mysnowflake",
         "Neev",
+        "numpy",
         "openpipeline",
         "pandoc",
         "pango",
         "parseable",
         "perc",
         "pipx",
         "pixbuf",
+        "pycache",
         "pylint",
         "pytest",
         "PYTHONPATH",
+        "pyyaml",
         "quiescing",
         "Ramdoss",
         "representer",
@@ -108,7 +113,9 @@
         "specversion",
         "spendings",
         "SQLERRM",
+        "sqlfluff",
         "SQLROWCOUNT",
+        "streamlit",
         "subrows",
         "subspan",
         "Suri",
@@ -143,7 +150,7 @@
     ],
     "focus.highlightRange": "none",
     "pylint.args": [
-        "--disable=C0111,C0415,C0301,W0621,W0150", 
+        "--disable=C0111,C0415,C0301,W0621,W0150",
         "--max-line-length=140"
     ],
     "vsicons.presets.jsonOfficial": true,

requirements.txt

@@ -48,4 +48,5 @@ flake8
 black
 sqlfluff
 yamllint
+flake8-docstrings
 #%:DEV

setup.cfg

@@ -0,0 +1,7 @@
+[flake8]
+docstring-convention = google
+max-line-length = 140
+ignore = F401,W503,E265,D400,D415,D107,D202,D205
+per-file-ignores =
+    test/*: D100,D101,D102,D103,D104,D105,D106,D107
+    src/*: D101

src/__init__.py

@@ -0,0 +1 @@
+"""This package initializes the src module and makes its submodules available for import."""

src/build/compile_bom.py

@@ -1,6 +1,4 @@
-"""
-Compiles all bom.yml files into a single Bill of Material (BOM) for SnowAgent
-"""
+"""Compiles all bom.yml files into a single Bill of Material (BOM) for SnowAgent"""
 
 #
 #
@@ -91,11 +89,11 @@ def _str_representer(dumper, data):
 
 
 def write_csv(data, filename):
-    # Extract the keys from the first dictionary as the header
+    """Extract the keys from the first dictionary as the header and write to a CSV file."""
     header = ["name", "type"] + list({key for d in data for key in d.keys() if key not in ["name", "type", "comment"]}) + ["comment"]
 
     # Open the CSV file for writing
-    with open(filename, "w", newline="") as csvfile:
+    with open(filename, "w", newline="", encoding="utf-8") as csvfile:
         import csv
 
         writer = csv.DictWriter(csvfile, fieldnames=header, delimiter="\t")

src/build/prepare_config.py

@@ -1,9 +1,4 @@
-"""
-Refactors config files to 3 levels of nesting (path, value, type) and redirects it to build/.
-Excludes keys starting with _ prefix.
-Args:
-    sys.argv[1:] (str): path to config files, specified when running `deploy.sh`
-"""
+"""Refactors config files to 3 levels of nesting (path, value, type) and redirects it to build/."""
 
 #
 #
@@ -92,10 +87,10 @@ def __recurse(config_element: Any, path: str = "") -> None:
 
 
 def _merge_json_files(*file_names: str) -> list:
-    """Enables to load given files with configuration JSONs one by one, and keep overriding existing keys (path)
+    """Loads and merges multiple configuration JSON files, overriding existing keys (by path) with values from later files.
 
     Returns:
-        list: the result is a merge of multiple configurations with values from last files in the list overriding those in previous positions
+        list: Merged configuration as a list, with later files' values overriding earlier ones for duplicate paths.
     """
     merged: dict[Any, Any] = {}
 
@@ -111,9 +106,9 @@ def _merge_json_files(*file_names: str) -> list:
 
 
 def main():
-    """
-    Refactors config files to 3 levels of nesting (path, value, type) and redirects it to build/.
+    """Refactors config files to 3 levels of nesting (path, value, type) and redirects it to build/.
     Excludes keys starting with _ prefix.
+
     Args:
         sys.argv[1:] (str): path to config files, specified when running `deploy.sh`
     """

src/build/update_docs.py

@@ -1,6 +1,4 @@
-"""
-Parses and builds the documentation from all instruments-def.yml, readme.md, config.md, and the default config files.
-"""
+"""Parses and builds the documentation from all instruments-def.yml, readme.md, config.md, and the default config files."""
 
 #
 #
@@ -100,8 +98,14 @@ def _generate_semantics_tables(json_data: Dict, plugin_name: str, no_global_cont
     for key in ["dimensions", "attributes", "metrics", "event_timestamps"]:
         if key in json_data and len(json_data[key]):
             __tables += f"### {key.replace('_', ' ').capitalize()} at the `{plugin_name}` plugin\n\n"
-            __tables += f"| Identifier {'| Name | Unit ' if key == 'metrics' else ''}| Description | Example {'| Context Name ' if no_global_context_name else ''}|\n"
-            __tables += f"|------------{'|------|------' if key == 'metrics' else ''}|-------------|---------{'|--------------' if no_global_context_name else ''}|\n"
+            __tables += (
+                f"| Identifier {'| Name | Unit ' if key == 'metrics' else ''}| Description | Example "
+                f"{'| Context Name ' if no_global_context_name else ''}|\n"
+            )
+            __tables += (
+                f"|------------{'|------|------' if key == 'metrics' else ''}|-------------|---------"
+                f"{'|--------------' if no_global_context_name else ''}|\n"
+            )
 
             for key_id, details in sorted(json_data[key].items()):
                 key_id = key_id.replace(".", ".​")
@@ -156,7 +160,8 @@ def _generate_plugins_info(dtagent_plugins_path: str) -> Tuple[str, List]:
                     __content += f"### {plugin_title} default configuration\n\n"
                     __content += (
                         "To disable this plugin, set `IS_DISABLED` to `true`.\n\n"
-                        "In case the global property `PLUGINS.DISABLED_BY_DEFAULT` is set to `true`, you need to explicitly set `IS_ENABLED` to `true` to enable selected plugins; `IS_DISABLED` is not checked then."
+                        "In case the global property `PLUGINS.DISABLED_BY_DEFAULT` is set to `true`, "
+                        "you need to explicitly set `IS_ENABLED` to `true` to enable selected plugins; `IS_DISABLED` is not checked then."
                         "\n\n"
                     )
                     __content += "```json\n" + _read_file(config_file_path) + "\n```\n\n"
@@ -228,7 +233,11 @@ def __contains_key(d: Dict, key: str) -> bool:
                         __content += f"[Show plugin description](#{plugin_name}_info_sec)\n\n"
 
                         if no_global_context_name:
-                            __content += f"This plugin delivers telemetry in multiple contexts. To filter by one of plugin's context names (reported as `{RUN_CONTEXT_KEY}`), please check the `Context Name` column below.\n\n"
+                            __content += (
+                                "This plugin delivers telemetry in multiple contexts. "
+                                f"To filter by one of plugin's context names (reported as `{RUN_CONTEXT_KEY}`), "
+                                "please check the `Context Name` column below.\n\n"
+                            )
                         else:
                             __content += (
                                 f'All telemetry delivered by this plugin is reported as `{RUN_CONTEXT_KEY} == "{plugin_name}"`.\n\n'
@@ -240,8 +249,7 @@ def __contains_key(d: Dict, key: str) -> bool:
 
 
 def _generate_appendix(appx_name: str) -> str:
-    """
-    Function to generate the appendix section for README.md
+    """Function to generate the appendix section for README.md
 
     Args:
         appx_name (str): Prefix for the appendix, e.g., "appx-a-appendix-name"
@@ -283,8 +291,7 @@ def _generate_appendix(appx_name: str) -> str:
 
 
 def _extract_appendix_info(header_file_path: str) -> Tuple[str, str]:
-    """
-    Extract appendix title and anchor from header markdown file.
+    """Extract appendix title and anchor from header markdown file.
 
     Args:
         header_file_path (str): Path to the header markdown file
@@ -306,8 +313,7 @@ def _extract_appendix_info(header_file_path: str) -> Tuple[str, str]:
 
 
 def generate_readme_content(dtagent_conf_path: str, dtagent_plugins_path: str) -> Tuple[str, str, str, str, str]:
-    """
-    Generates readme from sources
+    """Generates readme from sources
 
     Returns:
         A tuple containing the content for: readme_full_content, readme_short_content, plugins_content, semantics_content, appendix_content

src/dtagent/__init__.py

@@ -1,6 +1,4 @@
-"""
-Init module for dtagent.
-"""
+"""Init module for dtagent."""
 
 ##region ------------------------------ IMPORTS  -----------------------------------------
 #
@@ -142,9 +140,7 @@ def _get_biz_events(self) -> BizEvents:
         return BizEvents(self._configuration)
 
     def _get_config(self, session: snowpark.Session) -> Configuration:
-        """
-        Wrapper around Configuration constructor call to re-use in Test
-        """
+        """Wrapper around Configuration constructor call to re-use in Test"""
         return Configuration(session)
 
     def report_execution_status(
@@ -185,7 +181,7 @@ def handle_interrupted_run(self, source, exec_id, original_error):
             LOG.error("Unable to report failed run due to: %s", str(e2))
 
     def teardown(self) -> None:
-        """wrapping up, shutting logger and tracer"""
+        """Wrapping up, shutting logger and tracer"""
         self._logs.shutdown_logger()
         self._spans.shutdown_tracer()
 

src/dtagent/agent.py

@@ -96,8 +96,7 @@ class DynatraceSnowAgent(AbstractDynatraceSnowAgentConnector):
     """Main DynatraceSnowAgent class managing plugins executions"""
 
     def process(self, sources: List, run_proc: bool = True) -> Dict[str, Union[Dict[str, int], str]]:
-        """
-        Starts plugins specified in sources executions
+        """Starts plugins specified in sources executions
 
         Args:
             sources (List): List of measurement sources (plugins) to execute
@@ -181,9 +180,7 @@ def process(self, sources: List, run_proc: bool = True) -> Dict[str, Union[Dict[
 
 
 def main(session: snowpark.Session, sources: List) -> dict:
-    """
-    MAIN entry to this stored procedure - this is where the fun begins
-    """
+    """MAIN entry to this stored procedure - this is where the fun begins"""
     agent = DynatraceSnowAgent(session)
     results = agent.process(sources)
     agent.teardown()

src/dtagent/config.py

@@ -48,8 +48,7 @@ class Configuration:
     }
 
     def __init__(self, session: snowpark.Session) -> Dict:
-        """
-        Returns configuration based on data in config_data.configuration and (currently) hardcoded list of instruments
+        """Returns configuration based on data in config_data.configuration and (currently) hardcoded list of instruments
 
         {
             'dt.token': YOUR_TOKEN,
@@ -68,7 +67,8 @@ def __init__(self, session: snowpark.Session) -> Dict:
             'instruments': {
                 'metrics': {
                     'snowflake.data.scanned_from_cache': {
-                        'description': 'The percentage of data scanned from the local disk cache. The value ranges from 0.0 to 1.0. Multiply by 100 to get a true percentage.',
+                        'description': 'The percentage of data scanned from the local disk cache.
+                                        The value ranges from 0.0 to 1.0. Multiply by 100 to get a true percentage.',
                         'unit': 'percent'
                     },
                     ....
@@ -91,13 +91,16 @@ def __init__(self, session: snowpark.Session) -> Dict:
         from dtagent.otel.spans import Spans  # COMPILE_REMOVE
 
         def __rewrite_with_types(config_df: dict) -> dict:
-            """
-            This function rewrites the pandas dataframe with config to a dict type and assigns desired types to fields.
+            """This function rewrites the pandas dataframe with config to a dict type and assigns desired types to fields.
+
             List format in configuration table should be as follows to work properly:
-                List values must start with `[` and end with `]`, all fields must be separated with `, `. Values within the list should be enclosed in double quotes (").
+                List values must start with `[` and end with `]`, all fields must be separated with `, `.
+                Values within the list should be enclosed in double quotes (").
                 All items within the list must be the same type.
+
             Args:
                 config_df (dict) - pandas dataframe with configuration table contents
+
             Returns:
                 processed_dict (dict) - dictionary with reformatted field types
             """
@@ -110,7 +113,8 @@ def __rewrite_with_types(config_df: dict) -> dict:
             return processed_dict
 
         def __unpack_prefixed_keys(config: dict, prefix: Optional[str] = None) -> dict:
-            """Traverses key path (dot separated) to unpack flat dict into a dictionary where only numbers, strings, or lists are in the values-leaves
+            """Traverses key path (dot separated) to unpack flat dict into a dictionary where only numbers, strings, or
+            lists are in the values-leaves.
 
             Args:
                 config (dict): Dictionary to unpack
@@ -198,9 +202,10 @@ def get(
         """Returns configuration value for the given key in either given context or for the given plugin name
 
         Args:
-            key (str): Configuration key for which to return value
-            context (str, optional): First level key under which a dict is kept as a value. Defaults to None.
-            plugin_name (str, optional): Name of the key under the "plugins" key in configs; dictionary is expected under that plugin name. Defaults to None.
+            key (str):                   Configuration key for which to return value
+            context (str, optional):     First level key under which a dict is kept as a value. Defaults to None.
+            plugin_name (str, optional): Name of the key under the "plugins" key in configs; dictionary is expected under that plugin name.
+                                         Defaults to None.
 
         Returns:
             any: Value for the given configuration key in given context or plugin.
@@ -222,9 +227,7 @@ def get(
         return return_key
 
     def get_last_measurement_update(self, session: snowpark.Session, source: str):
-        """
-        Checks STATUS.PROCESSED_MEASUREMENTS_LOG to get last update for the given source
-        """
+        """Checks STATUS.PROCESSED_MEASUREMENTS_LOG to get last update for the given source"""
         from dtagent.util import _get_timestamp_in_sec  # COMPILE_REMOVE
 
         last_ts = session.sql(f"select APP.F_LAST_PROCESSED_TS('{source}');").collect()[0][0]

src/dtagent/connector.py

@@ -105,12 +105,10 @@
 
 
 class TelemetrySender(AbstractDynatraceSnowAgentConnector, Plugin):
-    """Telemetry sender class delivers possibility of sending custom data
-    from Snowflake to Grail, not being limited by plugins."""
+    """Telemetry sender class delivers possibility of sending custom data from Snowflake to Grail, not being limited by plugins."""
 
     def __init__(self, session: snowpark.Session, params: dict, exec_id: str) -> None:
-        """
-        Initialization for TelemetrySender class.
+        """Initialization for TelemetrySender class.
 
         Args:
             session (snowpark.Session): snowflake snowpark session
@@ -142,7 +140,7 @@ def __init__(self, session: snowpark.Session, params: dict, exec_id: str) -> Non
         self.__context = get_context_name_and_run_id(plugin_name=self._plugin_name, context_name=self.__context_name, run_id=exec_id)
 
     def process(self, run_id: str, run_proc: bool = True) -> Dict[str, int]:
-        """we don't use it but Plugin marks it as abstract"""
+        """We don't use it but Plugin marks it as abstract"""
 
         return {}
 
@@ -313,9 +311,7 @@ def send_data(self, source_data: Union[str, dict, list]) -> Dict[str, int]:
 
 
 def main(session: snowpark.Session, source: Union[str, dict, list], params: dict) -> str:
-    """
-    MAIN entry to this stored procedure - this is where the fun begins
-    """
+    """MAIN entry to this stored procedure - this is where the fun begins"""
     exec_id = str(uuid.uuid4().hex)
     sender = TelemetrySender(session, params, exec_id)
     try: