Merge pull request #801 from aboutcode-org/maven-fix

Fix maven pipeline failure when there is no new increment to mine

Author: keshav-space

minecode/management/commands/defederate_packages.py

@@ -75,6 +75,4 @@ def handle(self, *args, **options):
             repo = repo_data.get("repo")
             for purl in yield_purls_from_yaml_files(repo.working_dir):
                 # TODO: use batch create for efficiency
-                package = packagedb_models.Package.objects.create(
-                    **purl.to_dict()
-                )
+                package = packagedb_models.Package.objects.create(**purl.to_dict())

minecode/management/commands/federate_packages.py

@@ -9,7 +9,6 @@
 
 import logging
 import sys
-from aboutcode import hashid
 
 from aboutcode.federated import DataFederation
 from scanpipe.pipes import federatedcode
@@ -69,7 +68,7 @@ def handle(self, *args, **options):
         # TODO: do something more efficient
         files_to_commit = []
         commit_batch = 1
-        files_per_commit=PACKAGE_BATCH_SIZE
+        files_per_commit = PACKAGE_BATCH_SIZE
         for package in packagedb_models.Package.objects.all():
             package_repo, datafile_path = data_cluster.get_datafile_repo_and_path(purl=package.purl)
             purl_file = pipes.write_packageurls_to_file(

minecode_pipelines/__init__.py

@@ -8,4 +8,4 @@
 #
 
 
-VERSION = "0.1.0"
+VERSION = "0.1.1"

minecode_pipelines/miners/cpan.py

@@ -35,15 +35,15 @@ def get_cpan_packages(cpan_repo=CPAN_REPO, logger=None):
     packages_archive = get_temp_file(file_name="cpan_packages", extension=".gz")
     packages_content = get_temp_file(file_name="cpan_packages", extension=".txt")
     response = requests.get(cpan_packages_url, stream=True)
-    with open(packages_archive, 'wb') as f:
+    with open(packages_archive, "wb") as f:
         for chunk in response.iter_content(chunk_size=8192):
             f.write(chunk)
 
     with gzip.open(packages_archive, "rb") as f_in:
         with open(packages_content, "wb") as f_out:
             f_out.writelines(f_in)
 
-    with open(packages_content, 'r', encoding='utf-8') as file:
+    with open(packages_content, encoding="utf-8") as file:
         packages_content = file.read()
 
     package_path_by_name = {}

minecode_pipelines/pipelines/__init__.py

@@ -196,7 +196,7 @@ def _mine_and_publish_packageurls(
 
         if len(checkout["file_to_commit"]) > batch_size:
             if logger:
-                logger(f"Trying to commit PackageURLs.")
+                logger("Trying to commit PackageURLs.")
             pipes.commit_and_push_checkout(
                 local_checkout=checkout,
                 commit_message=commit_msg_func(checkout["commit_count"] + 1),

minecode_pipelines/pipelines/mine_cpan.py

@@ -21,10 +21,8 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 
-from minecode_pipelines import pipes
 from minecode_pipelines.pipes import cpan
 from minecode_pipelines.pipelines import MineCodeBasePipeline
-from scanpipe.pipes import federatedcode
 
 
 class MineCpan(MineCodeBasePipeline):
@@ -57,4 +55,3 @@ def mine_packageurls(self):
             package_path_by_name=self.cpan_packages_path_by_name,
             logger=self.log,
         )
-

minecode_pipelines/pipelines/mine_maven.py

@@ -43,6 +43,7 @@ def steps(cls):
             cls.fetch_federation_config,
             cls.fetch_checkpoint_and_maven_index,
             cls.mine_and_publish_maven_packageurls,
+            cls.save_check_point,
             cls.delete_working_dir,
         )
 
@@ -59,7 +60,10 @@ def fetch_checkpoint_and_maven_index(self):
 
         last_incremental = checkpoint.get("last_incremental")
         self.log(f"last_incremental: {last_incremental}")
-        self.maven_nexus_collector = maven.MavenNexusCollector(last_incremental=last_incremental)
+        self.maven_nexus_collector = maven.MavenNexusCollector(
+            last_incremental=last_incremental,
+            logger=self.log,
+        )
 
     def mine_and_publish_maven_packageurls(self):
         _mine_and_publish_packageurls(
@@ -71,7 +75,6 @@ def mine_and_publish_maven_packageurls(self):
             append_purls=self.append_purls,
             commit_msg_func=self.commit_message,
             logger=self.log,
-            checkpoint_func=self.save_check_point,
         )
 
     def save_check_point(self):

minecode_pipelines/pipelines/mine_npm.py

@@ -48,9 +48,9 @@ def steps(cls):
 
     def mine_npm_packages(self):
         """Mine npm package names from npm indexes or checkpoint."""
-        (
-            self.npm_packages, self.state, self.last_seq, self.config_repo
-        ) = npm.mine_npm_packages(logger=self.log)
+        (self.npm_packages, self.state, self.last_seq, self.config_repo) = npm.mine_npm_packages(
+            logger=self.log
+        )
 
     def get_npm_packages_to_sync(self):
         """Get npm packages which needs to be synced using checkpoint."""

minecode_pipelines/pipes/__init__.py

@@ -103,7 +103,9 @@ def get_mined_packages_from_checkpoint(config_repo, checkpoint_path):
     return checkpoint.get("packages_mined", [])
 
 
-def update_mined_packages_in_checkpoint(packages, config_repo, cloned_repo, checkpoint_path, logger=None):
+def update_mined_packages_in_checkpoint(
+    packages, config_repo, cloned_repo, checkpoint_path, logger=None
+):
     mined_packages = get_mined_packages_from_checkpoint(
         config_repo=config_repo,
         checkpoint_path=checkpoint_path,

minecode_pipelines/pipes/cran.py

@@ -22,9 +22,7 @@
 
 import json
 from pathlib import Path
-from typing import Iterable
-from typing import Tuple
-from typing import List
+from collections.abc import Iterable
 
 import requests
 from packageurl import PackageURL
@@ -49,7 +47,7 @@ def fetch_cran_db(working_path, logger) -> Path:
     return output_path
 
 
-def mine_cran_packageurls(db_path: Path) -> Iterable[Tuple[str, List[str]]]:
+def mine_cran_packageurls(db_path: Path) -> Iterable[tuple[str, list[str]]]:
     """
     Extract package names and their versions from a CRAN DB JSON file.
     Yields a tuple: (base_purl, list_of_purls)