Merge branch 'main' into version-up

Author: babenek

.ci/benchmark.txt

@@ -1,4 +1,4 @@
 [flake8]
 max-line-length = 120
-extend-ignore = E203,E303,E131,E402
+extend-ignore = E402
 per-file-ignores = __init__.py:F401

.flake8

@@ -31,7 +31,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 22b3f3789b3c2078d7dec12a6d210e11c1cc697c
+          ref: 5c93f142f9c28c03ff35ee0862dc819b73cdb673
 
       - name: Markup hashing
         run: |
@@ -87,7 +87,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 22b3f3789b3c2078d7dec12a6d210e11c1cc697c
+          ref: 5c93f142f9c28c03ff35ee0862dc819b73cdb673
 
       - name: Markup hashing
         run: |
@@ -190,7 +190,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 22b3f3789b3c2078d7dec12a6d210e11c1cc697c
+          ref: 5c93f142f9c28c03ff35ee0862dc819b73cdb673
 
       - name: Markup hashing
         run: |
@@ -378,7 +378,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 22b3f3789b3c2078d7dec12a6d210e11c1cc697c
+          ref: 5c93f142f9c28c03ff35ee0862dc819b73cdb673
 
       - name: Markup hashing
         run: |

.github/workflows/benchmark.yml

@@ -89,19 +89,19 @@ jobs:
 
       - name: Analysing the code with pylint and minimum Python version 3.9
         if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
-        run: pylint --py-version=3.9 --errors-only credsweeper
+        run: pylint --py-version=3.9 --verbose credsweeper
 
       - name: Analysing the code with pylint and minimum Python version 3.10
         if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
-        run: pylint --py-version=3.10 --errors-only credsweeper
+        run: pylint --py-version=3.10 --verbose credsweeper
 
       - name: Analysing the code with pylint and minimum Python version 3.11
         if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
-        run: pylint --py-version=3.11 --errors-only credsweeper
+        run: pylint --py-version=3.11 --verbose credsweeper
 
       - name: Analysing the code with pylint and minimum Python version 3.12
         if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
-        run: pylint --py-version=3.12 --errors-only credsweeper
+        run: pylint --py-version=3.12 --verbose credsweeper
 
       # # # mypy
 
@@ -120,13 +120,6 @@ jobs:
         run: |
           mypy --config-file .mypy.ini --python-version=3.12 credsweeper
 
-      # # # documentation
-
-      - name: Analysing the code with pylint for NEW missed docstrings of classes or functions
-        if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
-        run: |
-          pylint --disable=E,R,W1203,W0718,C0114,C0103,C0303,C0412,C0413,C0415,C0200,C0201,C0325 --verbose credsweeper
-
       # # # Documentation check
 
       - name: Test for creation sphinx documentations
@@ -138,20 +131,6 @@ jobs:
           cd source
           python -m sphinx -T -E -b html -d _build/doctrees -D language=en . ./_html
 
-      # # # yapf
-
-      - name: Check project style
-        if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
-        run: |
-          for f in credsweeper tests docs experiment; do
-              yapf --style .style.yapf --recursive --in-place --parallel $f
-          done
-          if [ 0 -ne $(git ls-files -m | wc -l) ]; then
-            git diff
-            echo "<- difference how to apply the style"
-            exit 1
-          fi
-
       # # # flake8
 
       - name: Analysing the code with flake8
@@ -198,6 +177,20 @@ jobs:
               exit 1
           fi
 
+      # # # yapf
+
+      - name: Check project style
+        if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
+        run: |
+          for f in credsweeper tests docs experiment; do
+              yapf --style .style.yapf --recursive --in-place --parallel $f
+          done
+          if [ 0 -ne $(git ls-files -m | wc -l) ]; then
+            git diff
+            echo "<- difference how to apply the style"
+            exit 1
+          fi
+
       # # # SECURITY.md check
 
       - name: SECURITY.md check

.github/workflows/check.yml

@@ -4,6 +4,7 @@
 warn_return_any = True
 warn_unused_configs = True
 strict_optional = False
+check_untyped_defs = True
 
 # Per-module options:
 

.mypy.ini

@@ -2,3 +2,6 @@
 max-line-length=120
 
 extension-pkg-allow-list=lxml
+
+[MESSAGES CONTROL]
+disable=R,W0718,W1203,C0415,C0413,C0103,C0114

.pylintrc

@@ -63,7 +63,7 @@ def logger_levels(log_level: str) -> str:
     Returns True if log_level UPPERCASE is one of keys
     """
     val = log_level.upper()
-    if any(val == i for i in Logger.LEVELS.keys()):
+    if val in Logger.LEVELS:
         return val
     raise ArgumentTypeError(f"Log level provided: {log_level} -- must be one of: {' | '.join(Logger.LEVELS.keys())}")
 

credsweeper/__main__.py

@@ -15,11 +15,13 @@
 from credsweeper.config import Config
 from credsweeper.credentials import Candidate, CredentialManager, CandidateKey
 from credsweeper.deep_scanner.deep_scanner import DeepScanner
+from credsweeper.file_handler.content_provider import ContentProvider
 from credsweeper.file_handler.diff_content_provider import DiffContentProvider
 from credsweeper.file_handler.file_path_extractor import FilePathExtractor
 from credsweeper.file_handler.abstract_provider import AbstractProvider
 from credsweeper.file_handler.text_content_provider import TextContentProvider
 from credsweeper.scanner import Scanner
+from credsweeper.ml_model.ml_validator import MlValidator
 from credsweeper.utils import Util
 
 logger = logging.getLogger(__name__)
@@ -94,7 +96,7 @@ def __init__(self,
             log_level: str - level for pool initializer according logging levels (UPPERCASE)
 
         """
-        self.pool_count: int = int(pool_count) if int(pool_count) > 1 else 1
+        self.pool_count: int = max(1, int(pool_count))
         if not (_severity := Severity.get(severity)):
             raise RuntimeError(f"Severity level provided: {severity}"
                                f" -- must be one of: {' | '.join([i.value for i in Severity])}")
@@ -123,9 +125,9 @@ def __init__(self,
         self.ml_config = ml_config
         self.ml_model = ml_model
         self.ml_providers = ml_providers
-        self.ml_validator = None
         self.__thrifty = thrifty
         self.__log_level = log_level
+        self.__ml_validator: Optional[MlValidator] = None
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -182,35 +184,22 @@ def _use_ml_validation(self) -> bool:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    # the import cannot be done on top due
-    # TypeError: cannot pickle 'onnxruntime.capi.onnxruntime_pybind11_state.InferenceSession' object
-    from credsweeper.ml_model import MlValidator
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     @property
     def ml_validator(self) -> MlValidator:
         """ml_validator getter"""
-        from credsweeper.ml_model import MlValidator
         if not self.__ml_validator:
-            self.__ml_validator: MlValidator = MlValidator(
+            self.__ml_validator = MlValidator(
                 threshold=self.ml_threshold,  #
                 ml_config=self.ml_config,  #
                 ml_model=self.ml_model,  #
                 ml_providers=self.ml_providers,  #
             )
-        assert self.__ml_validator, "self.__ml_validator was not initialized"
+        if not self.__ml_validator:
+            raise RuntimeError("MlValidator was not initialized!")
         return self.__ml_validator
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    @ml_validator.setter
-    def ml_validator(self, _ml_validator: Optional[MlValidator]) -> None:
-        """ml_validator setter"""
-        self.__ml_validator = _ml_validator
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     @staticmethod
     def pool_initializer(log_kwargs) -> None:
         """Ignore SIGINT in child processes."""
@@ -219,20 +208,6 @@ def pool_initializer(log_kwargs) -> None:
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    @property
-    def config(self) -> Config:
-        """config getter"""
-        return self.__config
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
-    @config.setter
-    def config(self, config: Config) -> None:
-        """config setter"""
-        self.__config = config
-
-    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
     def run(self, content_provider: AbstractProvider) -> int:
         """Run an analysis of 'content_provider' object.
 
@@ -241,9 +216,10 @@ def run(self, content_provider: AbstractProvider) -> int:
 
         """
         _empty_list: Sequence[Union[DiffContentProvider, TextContentProvider]] = []
-        file_extractors: Sequence[Union[DiffContentProvider, TextContentProvider]] = \
-            content_provider.get_scannable_files(self.config) if content_provider else _empty_list
-        logger.info(f"Start Scanner for {len(file_extractors)} providers")
+        file_extractors = content_provider.get_scannable_files(self.config) if content_provider else _empty_list
+        if not file_extractors:
+            logger.info(f"No scannable targets for {len(content_provider.paths)} paths")
+            return 0
         self.scan(file_extractors)
         self.post_processing()
         # PatchesProvider has the attribute. Circular import error appears with using the isinstance
@@ -260,7 +236,7 @@ def scan(self, content_providers: Sequence[Union[DiffContentProvider, TextConten
             content_providers: file objects to scan
 
         """
-        if 1 < self.pool_count:
+        if 1 < self.pool_count and 1 < len(content_providers):
             self.__multi_jobs_scan(content_providers)
         else:
             self.__single_job_scan(content_providers)
@@ -269,6 +245,7 @@ def scan(self, content_providers: Sequence[Union[DiffContentProvider, TextConten
 
     def __single_job_scan(self, content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> None:
         """Performs scan in main thread"""
+        logger.info(f"Scan for {len(content_providers)} providers")
         all_cred = self.files_scan(content_providers)
         self.credential_manager.set_credentials(all_cred)
 
@@ -284,12 +261,14 @@ def __multi_jobs_scan(self, content_providers: Sequence[Union[DiffContentProvide
             if "SILENCE" == self.__log_level:
                 logging.addLevelName(60, "SILENCE")
             log_kwargs["level"] = self.__log_level
-        with multiprocessing.get_context("spawn").Pool(processes=self.pool_count,
-                                                       initializer=self.pool_initializer,
+        pool_count = min(self.pool_count, len(content_providers))
+        logger.info(f"Scan in {pool_count} processes for {len(content_providers)} providers")
+        with multiprocessing.get_context("spawn").Pool(processes=pool_count,
+                                                       initializer=CredSweeper.pool_initializer,
                                                        initargs=(log_kwargs, )) as pool:
             try:
-                for scan_results in pool.imap_unordered(self.files_scan, (content_providers[x::self.pool_count]
-                                                                          for x in range(self.pool_count))):
+                for scan_results in pool.imap_unordered(self.files_scan,
+                                                        (content_providers[x::pool_count] for x in range(pool_count))):
                     for cred in scan_results:
                         self.credential_manager.add_credential(cred)
             except KeyboardInterrupt:
@@ -301,9 +280,7 @@ def __multi_jobs_scan(self, content_providers: Sequence[Union[DiffContentProvide
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def files_scan(
-            self,  #
-            content_providers: Sequence[Union[DiffContentProvider, TextContentProvider]]) -> List[Candidate]:
+    def files_scan(self, content_providers: Sequence[ContentProvider]) -> List[Candidate]:
         """Auxiliary method for scan one sequence"""
         all_cred: List[Candidate] = []
         for provider in content_providers:
@@ -316,7 +293,7 @@ def files_scan(
 
     # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-    def file_scan(self, content_provider: Union[DiffContentProvider, TextContentProvider]) -> List[Candidate]:
+    def file_scan(self, content_provider: ContentProvider) -> List[Candidate]:
         """Run scanning of file from 'file_provider'.
 
         Args:

credsweeper/app.py

@@ -96,10 +96,6 @@ class Chars(Enum):
     ASCII_PRINTABLE = string.printable
 
 
-ENTROPY_LIMIT_BASE64 = 4.5
-ENTROPY_LIMIT_BASE3x = 3
-
-
 class GroupType(Enum):
     """Group type - used in Group constructor for load predefined set of filters"""
     KEYWORD = "keyword"
@@ -148,7 +144,8 @@ class DiffRowType(Enum):
 CHUNK_STEP_SIZE = CHUNK_SIZE - OVERLAP_SIZE
 # ML hunk size to limit of variable or value size and get substring near value
 ML_HUNK = 80
-""" values according https://docs.python.org/3/library/codecs.html """
+
+# values according https://docs.python.org/3/library/codecs.html
 UTF_8 = "utf_8"
 UTF_16 = "utf_16"
 LATIN_1 = "latin_1"

credsweeper/common/constants.py

@@ -24,7 +24,7 @@ def __eq__(self, other):
         return self.key == other.key
 
     def __ne__(self, other):
-        return not (self == other)
+        return not bool(self == other)
 
     def __repr__(self) -> str:
         return f"{self.key}:{self.__line}"