Gaiaplat 1838 : Applying PyLint and Resolving any warnings from Docker_Dev (#1360)

* GAIAPLAT-1838 - Starting with initial  GDEV

https://gaiaplatform.atlassian.net/browse/GAIAPLAT-1838

* Fixing Pylint Warnings

Author: JackAtGaia

.pre-commit-config.yaml

@@ -185,5 +185,5 @@ repos:
   rev: v2.11.1
   hooks:
     - id: pylint
-      exclude: ^(dev_tools/gdev/|dev_tools/docker_dev/|production/tools/tests/gaiat/lit.cfg.py|production/tests/python/ps_mem.py)
+      exclude: ^(dev_tools/gdev/|production/tools/tests/gaiat/lit.cfg.py|production/tests/python/ps_mem.py)
       args: [--disable=R0801]

dev_tools/docker_dev/gdev/__main__.py

@@ -15,6 +15,9 @@
 from gdev.main import DockerDev
 
 def main() -> int:
+    """
+    Module main entry point into the application.
+    """
     DockerDev().main()
     return 0
 

dev_tools/docker_dev/gdev/cmd/gen/_abc/build.py

@@ -5,17 +5,21 @@
 # All rights reserved.
 #############################################
 
+"""
+Module to help build a docker image from its component pieces.
+"""
+
 from __future__ import annotations
 from abc import ABC, abstractmethod
 import os
-from typing import Awaitable, Callable, Iterable, Mapping, Union
+from typing import Iterable, Mapping
 
-from gdev.custom.pathlib import Path
+from gdev.custom.gaia_path import GaiaPath
 from gdev.dependency import Dependency
 from gdev.host import Host
 from gdev.third_party.atools import memoize
-from .cfg import GenAbcCfg
-from .dockerfile import GenAbcDockerfile
+from gdev.cmd.gen._abc.cfg import GenAbcCfg
+from gdev.cmd.gen._abc.dockerfile import GenAbcDockerfile
 
 
 # We require buildkit support for inline caching of multi-stage dockerfiles. It's also way faster
@@ -44,15 +48,6 @@ def dockerfile(self) -> GenAbcDockerfile:
         """
         raise NotImplementedError
 
-    # TODO: Used?
-    @memoize
-    def __get_actual_git_hash(self) -> str:
-        actual_git_hash = self._get_actual_label_value('GitHash')
-
-        self.log.debug(f'{actual_git_hash = }')
-
-        return actual_git_hash
-
     @memoize
     def _get_actual_label_value(self, name: str) -> str:
         """
@@ -72,18 +67,20 @@ def _get_actual_label_value(self, name: str) -> str:
     @memoize
     def _get_actual_label_value_by_name(self) -> Mapping[str, str]:
         """
-        Get the hash of an image with the actual label values that are called for by the configuration.
+        Get the hash of an image with the actual label values that are called
+        for by the configuration.
         """
         return {'GitHash': self._get_actual_label_value(name='GitHash')}
 
     @memoize
     def get_actual_label_value_by_name(self) -> Mapping[str, str]:
         """
-        Get the hash of an image with the actual label values that are called for by the configuration.
+        Get the hash of an image with the actual label values that are called
+        for by the configuration.
         """
 
         actual_label_value_by_name = self._get_actual_label_value_by_name()
-        self.log.debug(f'{actual_label_value_by_name = }')
+        self.log.debug('actual_label_value_by_name = %s', actual_label_value_by_name)
         return actual_label_value_by_name
 
     @memoize
@@ -102,7 +99,7 @@ def inner(dockerfile: GenAbcDockerfile) -> Iterable[str]:
 
         base_build_names = tuple(inner(self.dockerfile))
 
-        self.log.debug(f'{base_build_names = }')
+        self.log.debug('base_build_names = %s', base_build_names)
 
         return base_build_names
 
@@ -119,7 +116,7 @@ def get_sha(self) -> str:
         else:
             sha = ''
 
-        self.log.debug(f'{sha = }')
+        self.log.debug('sha = %s', sha)
 
         return sha
 
@@ -131,44 +128,18 @@ def get_tag(self) -> str:
 
         tag = f'{self.dockerfile.get_name()}:latest'
 
-        self.log.debug(f'{tag = }')
+        self.log.debug('tag = %s', tag)
 
         return tag
 
-    # TODO unused?
-    @memoize
-    def __get_uncommitted(self) -> str:
-        seen_dockerfiles = set()
-
-        def inner(dockerfile: GenAbcDockerfile) -> Iterable[Path]:
-            paths = set()
-            if dockerfile not in seen_dockerfiles:
-                seen_dockerfiles.add(dockerfile)
-                paths.add( Path.repo() / Path(dockerfile.options.target))
-                for input_dockerfile in dockerfile.get_input_dockerfiles():
-                    paths |= inner(input_dockerfile)
-            return paths
-
-        uncommitted = '\n'.join(
-            Host.execute_and_get_lines_sync(
-                f'git status --short ' + ' '.join(
-                    f'{path.parent}' for path in inner(self.dockerfile)
-                )
-            )
-        )
-
-        self.log.debug(f'{uncommitted = }')
-
-        return uncommitted
-
     @memoize
     def __get_wanted_git_hash(self) -> str:
         """
         Request that GIT provides information about the SHA of the HEAD node.
         """
         wanted_git_hash = Host.execute_and_get_line_sync('git rev-parse HEAD')
 
-        self.log.debug(f'{wanted_git_hash = }')
+        self.log.debug('wanted_git_hash = %s', wanted_git_hash)
 
         return wanted_git_hash
 
@@ -185,7 +156,7 @@ def get_wanted_label_value_by_name(self) -> Mapping[str, str]:
         """
         wanted_label_value_by_name = self._get_wanted_label_value_by_name()
 
-        self.log.debug(f'{wanted_label_value_by_name = }')
+        self.log.debug('wanted_label_value_by_name = %s', wanted_label_value_by_name)
 
         return wanted_label_value_by_name
 
@@ -203,8 +174,7 @@ def main(self) -> None:
                     for base_build_name in self.__get_base_build_names()])
             cached_images = f"--cache-from {cached_images}"
 
-        # TODO query remotely for cached build sources.
-        self.log.info(f'Creating image "{self.get_tag()}"')
+        self.log.info('Creating image "%s"', self.get_tag())
         Host.execute_sync(
             f'docker buildx build'
             f' -f {self.dockerfile.path}'
@@ -229,10 +199,13 @@ def main(self) -> None:
 
             f' {cached_images}'
 
-            f' {Path.repo()}'
+            f' {GaiaPath.repo()}'
         )
-        Host.execute_sync(f'docker image prune -f')
+        Host.execute_sync('docker image prune -f')
 
+    # pylint: disable=import-outside-toplevel
+    #
+    # Required to resolve cyclical dependency issues.
     @memoize
     def cli_entrypoint(self) -> None:
         """
@@ -242,7 +215,8 @@ def cli_entrypoint(self) -> None:
         if not self.options.mixins:
             build = self
         else:
-            from .._custom.build import GenCustomBuild
+            from gdev.cmd.gen._custom.build import GenCustomBuild
             build = GenCustomBuild(options=self.options, base_build=self)
 
         build.run()
+    # pylint: enable=import-outside-toplevel

dev_tools/docker_dev/gdev/cmd/gen/_abc/cfg.py

@@ -13,32 +13,31 @@
 import re
 from typing import FrozenSet, Iterable, Pattern
 
-from gdev.custom.pathlib import Path
+from gdev.custom.gaia_path import GaiaPath
 from gdev.dependency import Dependency
 from gdev.third_party.atools import memoize
 
-
 class GenAbcCfg(Dependency, ABC):
     """
     Class to parse the target gdev.cfg for build rules.
     """
 
     @property
     @memoize
-    def path(self) -> Path:
+    def path(self) -> GaiaPath:
         """
         Determine the path to the configuration file that we are observing.
         """
-        path = Path.repo() / self.options.target / 'gdev.cfg'
-        self.log.debug(f'{path = }')
+        path = GaiaPath.repo() / self.options.target / 'gdev.cfg'
+        self.log.debug('path = %s', path)
         return path
 
     @property
     def section_name(self) -> str:
         """
         Determine the section name in the configuration based on the type of the class.
         """
-        return Path(getfile(type(self))).parent.name.strip('_')
+        return GaiaPath(getfile(type(self))).parent.name.strip('_')
 
     @memoize
     def __get_begin_pattern(self) -> Pattern:
@@ -47,30 +46,36 @@ def __get_begin_pattern(self) -> Pattern:
         """
 
         begin_pattern = re.compile(fr'^\[{self.section_name}]$')
-        self.log.debug(f'{begin_pattern = }')
+        self.log.debug('begin_pattern = %s', begin_pattern)
         return begin_pattern
 
     @memoize
     def __get_end_pattern(self) -> Pattern:
         """
         Get the regex pattern that identifies the end of the section.
         """
-        end_pattern = re.compile(fr'^(# .*|)\[.+]$')
-        self.log.debug(f'{end_pattern = }')
+        end_pattern = re.compile(r'^(# .*|)\[.+]$')
+        self.log.debug('end_pattern = %s', end_pattern)
         return end_pattern
 
+
+    # pylint: disable=eval-used
     @staticmethod
     @memoize
-    def get_lines(cfg_enables: FrozenSet[str], path: Path) -> Iterable[str]:
+    def get_lines(cfg_enables: FrozenSet[str], path: GaiaPath) -> Iterable[str]:
         """
-        Get the various lines for the section with the inline notations like `{enable_if('CI_GitHub')}`
-        replaced with `# enable by setting "{'CI_GitHub'}": `.
+        Get the various lines for the section with the inline notations like
+        `{enable_if('CI_GitHub')}` replaced with `# enable by setting "{'CI_GitHub'}": `.
         """
-        return tuple([
+        # Per suggestions from https://realpython.com/python-eval-function/:
+        # - `__locals` field set to an empty dictionary
+        # - `__globals` field set to contain empty `__builtins__` item
+
+        return tuple((
             eval(
                 f'fr""" {line} """',
                 {
-                    'build_dir': Path.build,
+                    'build_dir': GaiaPath.build,
                     'enable_if': lambda enable:
                         '' if enable in cfg_enables
                         else f'# enable by setting "{enable}": ',
@@ -89,15 +94,17 @@ def get_lines(cfg_enables: FrozenSet[str], path: Path) -> Iterable[str]:
                     'enable_if_not_all': lambda *enables:
                         '' if not (set(enables) in cfg_enables)
                         else f'# enable by not setting all of "{sorted(set(enables))}": ',
-                    'source_dir': Path.source
-                }
+                    'source_dir': GaiaPath.source,
+                    "__builtins__": {}
+                }, {}
             )[1:-1]
             for line in (GenAbcCfg.__get_raw_text(path)).splitlines()
-        ])
+        ))
+    # pylint: enable=eval-used
 
     @staticmethod
     @memoize
-    def __get_raw_text(path: Path) -> str:
+    def __get_raw_text(path: GaiaPath) -> str:
         if not path.is_file():
             raise Dependency.Abort(f'File "<repo_root>/{path.context()}" must exist.')
 
@@ -142,7 +149,7 @@ def get_section_lines(self) -> Iterable[str]:
 
         section_lines = tuple(section_lines)
 
-        self.log.debug(f'{section_lines = }')
+        self.log.debug('section_lines = %s', section_lines)
 
         return section_lines