Prepare code for per-directory configuration files

- Add opportunity to open checkers per-file, so they can use values from local config during opening
- Save command line arguments to apply them on top of each new config
- More accurate verbose messages about config files
- Enable finding config files in arbitrary directories
- Add opportunity to call linter._astroid_module_checker per file in single-process mode
- Collect stats from several calls of linter._astroid_module_checker in single-process mode
- Extend linter._get_namespace_for_file to return the path from which namespace was created

Author: Aleksey Petryankin

.pyenchant_pylint_custom_dict.txt

@@ -61,6 +61,7 @@ codecs
 col's
 conf
 config
+configs
 const
 Const
 contextlib
@@ -310,11 +311,13 @@ str
 stringified
 subclasses
 subcommands
+subconfigs
 subdicts
 subgraphs
 sublists
 submodule
 submodules
+subpackage
 subparsers
 subparts
 subprocess

pylint/config/arguments_manager.py

@@ -81,6 +81,9 @@ def __init__(
         self._directory_namespaces: DirectoryNamespaceDict = {}
         """Mapping of directories and their respective namespace objects."""
 
+        self._cli_args: list[str] = []
+        """Options that were passed as command line arguments and have highest priority."""
+
     @property
     def config(self) -> argparse.Namespace:
         """Namespace for all options."""
@@ -226,6 +229,8 @@ def _parse_command_line_configuration(
     ) -> list[str]:
         """Parse the arguments found on the command line into the namespace."""
         arguments = sys.argv[1:] if arguments is None else arguments
+        if not self._cli_args:
+            self._cli_args = list(arguments)
 
         self.config, parsed_args = self._arg_parser.parse_known_args(
             arguments, self.config

pylint/config/config_file_parser.py

@@ -106,7 +106,7 @@ def parse_config_file(
             raise OSError(f"The config file {file_path} doesn't exist!")
 
         if verbose:
-            print(f"Using config file {file_path}", file=sys.stderr)
+            print(f"Loading config file {file_path}", file=sys.stderr)
 
         if file_path.suffix == ".toml":
             return _RawConfParser.parse_toml_file(file_path)

pylint/config/config_initialization.py

@@ -23,6 +23,7 @@
     from pylint.lint import PyLinter
 
 
+# pylint: disable = too-many-statements
 def _config_initialization(
     linter: PyLinter,
     args_list: list[str],
@@ -82,6 +83,9 @@ def _config_initialization(
     args_list = _order_all_first(args_list, joined=True)
     parsed_args_list = linter._parse_command_line_configuration(args_list)
 
+    # save Runner.verbose to make this preprocessed option visible from other modules
+    linter.config.verbose = verbose_mode
+
     # Remove the positional arguments separator from the list of arguments if it exists
     try:
         parsed_args_list.remove("--")
@@ -141,7 +145,8 @@ def _config_initialization(
     linter._parse_error_mode()
 
     # Link the base Namespace object on the current directory
-    linter._directory_namespaces[Path(".").resolve()] = (linter.config, {})
+    if Path(".").resolve() not in linter._directory_namespaces:
+        linter._directory_namespaces[Path(".").resolve()] = (linter.config, {})
 
     # parsed_args_list should now only be a list of inputs to lint.
     # All other options have been removed from the list.

pylint/config/find_default_config_files.py

@@ -64,17 +64,19 @@ def _cfg_has_config(path: Path | str) -> bool:
     return any(section.startswith("pylint.") for section in parser.sections())
 
 
-def _yield_default_files() -> Iterator[Path]:
+def _yield_default_files(basedir: Path | str = ".") -> Iterator[Path]:
     """Iterate over the default config file names and see if they exist."""
+    basedir = Path(basedir)
     for config_name in CONFIG_NAMES:
+        config_file = basedir / config_name
         try:
-            if config_name.is_file():
-                if config_name.suffix == ".toml" and not _toml_has_config(config_name):
+            if config_file.is_file():
+                if config_file.suffix == ".toml" and not _toml_has_config(config_file):
                     continue
-                if config_name.suffix == ".cfg" and not _cfg_has_config(config_name):
+                if config_file.suffix == ".cfg" and not _cfg_has_config(config_file):
                     continue
 
-                yield config_name.resolve()
+                yield config_file.resolve()
         except OSError:
             pass
 
@@ -142,3 +144,8 @@ def find_default_config_files() -> Iterator[Path]:
             yield Path("/etc/pylintrc").resolve()
     except OSError:
         pass
+
+
+def find_subdirectory_config_files(basedir: Path | str) -> Iterator[Path]:
+    """Find config file in arbitrary subdirectory."""
+    yield from _yield_default_files(basedir)

pylint/lint/pylinter.py

@@ -66,7 +66,7 @@
     ModuleDescriptionDict,
     Options,
 )
-from pylint.utils import ASTWalker, FileState, LinterStats, utils
+from pylint.utils import ASTWalker, FileState, LinterStats, merge_stats, utils
 
 MANAGER = astroid.MANAGER
 
@@ -317,6 +317,7 @@ def __init__(
 
         # Attributes related to stats
         self.stats = LinterStats()
+        self.all_stats: list[LinterStats] = []
 
         # Attributes related to (command-line) options and their parsing
         self.options: Options = options + _make_linter_options(self)
@@ -665,12 +666,12 @@ def check(self, files_or_modules: Sequence[str]) -> None:
                     "Missing filename required for --from-stdin"
                 )
 
-        extra_packages_paths = list(
-            {
+        extra_packages_paths_set = set()
+        for file_or_module in files_or_modules:
+            extra_packages_paths_set.add(
                 discover_package_path(file_or_module, self.config.source_roots)
-                for file_or_module in files_or_modules
-            }
-        )
+            )
+        extra_packages_paths = list(extra_packages_paths_set)
 
         # TODO: Move the parallel invocation into step 3 of the checking process
         if not self.config.from_stdin and self.config.jobs > 1:
@@ -693,13 +694,12 @@ def check(self, files_or_modules: Sequence[str]) -> None:
                 fileitems = self._iterate_file_descrs(files_or_modules)
                 data = None
 
-        # The contextmanager also opens all checkers and sets up the PyLinter class
         with augmented_sys_path(extra_packages_paths):
+            # 2) Get the AST for each FileItem
+            ast_per_fileitem = self._get_asts(fileitems, data)
+            # 3) Lint each ast
+            # The contextmanager also opens all checkers and sets up the PyLinter class
             with self._astroid_module_checker() as check_astroid_module:
-                # 2) Get the AST for each FileItem
-                ast_per_fileitem = self._get_asts(fileitems, data)
-
-                # 3) Lint each ast
                 self._lint_files(ast_per_fileitem, check_astroid_module)
 
     def _get_asts(
@@ -710,6 +710,7 @@ def _get_asts(
 
         for fileitem in fileitems:
             self.set_current_module(fileitem.name, fileitem.filepath)
+            self._set_astroid_options()
 
             try:
                 ast_per_fileitem[fileitem] = self.get_ast(
@@ -735,13 +736,14 @@ def check_single_file_item(self, file: FileItem) -> None:
 
         initialize() should be called before calling this method
         """
+        self.set_current_module(file.name, file.filepath)
         with self._astroid_module_checker() as check_astroid_module:
             self._check_file(self.get_ast, check_astroid_module, file)
 
     def _lint_files(
         self,
         ast_mapping: dict[FileItem, nodes.Module | None],
-        check_astroid_module: Callable[[nodes.Module], bool | None],
+        check_astroid_module: Callable[[nodes.Module], bool | None] | None,
     ) -> None:
         """Lint all AST modules from a mapping.."""
         for fileitem, module in ast_mapping.items():
@@ -760,12 +762,17 @@ def _lint_files(
                     )
                 else:
                     self.add_message("fatal", args=msg, confidence=HIGH)
+        # current self.stats is needed in merge - it contains stats from last module
+        finished_run_stats = merge_stats([*self.all_stats, self.stats])
+        # after _lint_files linter.stats is aggregate stats from all modules, like after check_parallel
+        self.all_stats = []
+        self.stats = finished_run_stats
 
     def _lint_file(
         self,
         file: FileItem,
         module: nodes.Module,
-        check_astroid_module: Callable[[nodes.Module], bool | None],
+        check_astroid_module: Callable[[nodes.Module], bool | None] | None,
     ) -> None:
         """Lint a file using the passed utility function check_astroid_module).
 
@@ -784,7 +791,13 @@ def _lint_file(
         self.current_file = module.file
 
         try:
-            check_astroid_module(module)
+            # call _astroid_module_checker after set_current_module, when
+            # self.config is the right config for current module
+            if check_astroid_module is None:
+                with self._astroid_module_checker() as local_check_astroid_module:
+                    local_check_astroid_module(module)
+            else:
+                check_astroid_module(module)
         except Exception as e:
             raise astroid.AstroidError from e
 
@@ -898,33 +911,44 @@ def _expand_files(
     def set_current_module(self, modname: str, filepath: str | None = None) -> None:
         """Set the name of the currently analyzed module and
         init statistics for it.
+
+        Save current stats before init to make sure no counters for
+        error, statement, etc are missed.
         """
         if not modname and filepath is None:
             return
         self.reporter.on_set_current_module(modname or "", filepath)
         self.current_name = modname
         self.current_file = filepath or modname
+        self.all_stats.append(self.stats)
+        self.stats = LinterStats()
         self.stats.init_single_module(modname or "")
 
         # If there is an actual filepath we might need to update the config attribute
         if filepath:
-            namespace = self._get_namespace_for_file(
+            config_path, namespace = self._get_namespace_for_file(
                 Path(filepath), self._directory_namespaces
             )
             if namespace:
-                self.config = namespace or self._base_config
+                self.config = namespace
+                if self.config.verbose:
+                    print(
+                        f"Using config file from {config_path} for {filepath}",
+                        file=sys.stderr,
+                    )
 
     def _get_namespace_for_file(
         self, filepath: Path, namespaces: DirectoryNamespaceDict
-    ) -> argparse.Namespace | None:
+    ) -> tuple[Path | None, argparse.Namespace | None]:
+        filepath = filepath.resolve()
         for directory in namespaces:
             if _is_relative_to(filepath, directory):
-                namespace = self._get_namespace_for_file(
+                _, namespace = self._get_namespace_for_file(
                     filepath, namespaces[directory][1]
                 )
                 if namespace is None:
-                    return namespaces[directory][0]
-        return None
+                    return directory, namespaces[directory][0]
+        return None, None
 
     @contextlib.contextmanager
     def _astroid_module_checker(
@@ -953,7 +977,7 @@ def _astroid_module_checker(
             rawcheckers=rawcheckers,
         )
 
-        # notify global end
+        # notify end of module if jobs>1, global end otherwise
         self.stats.statement = walker.nbstatements
         for checker in reversed(_checkers):
             checker.close()
@@ -1068,22 +1092,26 @@ def _check_astroid_module(
         walker.walk(node)
         return True
 
-    def open(self) -> None:
-        """Initialize counters."""
+    def _set_astroid_options(self) -> None:
+        """Pass some config values to astroid.MANAGER object."""
         MANAGER.always_load_extensions = self.config.unsafe_load_any_extension
         MANAGER.max_inferable_values = self.config.limit_inference_results
         MANAGER.extension_package_whitelist.update(self.config.extension_pkg_allow_list)
         if self.config.extension_pkg_whitelist:
             MANAGER.extension_package_whitelist.update(
                 self.config.extension_pkg_whitelist
             )
-        self.stats.reset_message_count()
+
+    def open(self) -> None:
+        """Initialize self as main checker for one or more modules."""
+        self._set_astroid_options()
 
     def generate_reports(self, verbose: bool = False) -> int | None:
         """Close the whole package /module, it's time to make reports !
 
         if persistent run, pickle results for later comparison
         """
+        self.config = self._base_config
         # Display whatever messages are left on the reporter.
         self.reporter.display_messages(report_nodes.Section())
         if not self.file_state._is_base_filestate: