Refactor debian mining pipeline for git deployment

Signed-off-by: Keshav Priyadarshi <[email protected]>

Author: keshav-space

minecode_pipelines/__init__.py

@@ -8,4 +8,4 @@
 #
 
 
-VERSION = "0.0.1b42"
+VERSION = "0.0.1b53"

minecode_pipelines/pipelines/mine_debian.py

@@ -20,36 +20,72 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import federatedcode
 
 from minecode_pipelines import pipes
+from minecode_pipelines.pipelines import MineCodeBasePipeline
+from minecode_pipelines.pipelines import _mine_and_publish_packageurls
 from minecode_pipelines.pipes import debian
 
 
-class MineDebian(Pipeline):
-    """
-    Mine all packageURLs from a Debian index and publish them to
-    a FederatedCode repo.
-    """
+class MineDebian(MineCodeBasePipeline):
+    """Mine PackageURLs from Debian index and publish them to FederatedCode."""
+
+    pipeline_config_repo = "https://github.com/aboutcode-data/minecode-pipelines-config/"
+    checkpoint_path = "debian/checkpoints.json"
+    append_purls = True
 
     @classmethod
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
-            cls.collect_packages_from_debian,
-            cls.delete_cloned_repos,
+            cls.create_federatedcode_working_dir,
+            cls.fetch_federation_config,
+            cls.fetch_checkpoint_and_debian_index,
+            cls.mine_and_publish_alpine_packageurls,
+            cls.save_check_point,
+            cls.delete_working_dir,
         )
 
-    def check_federatedcode_eligibility(self):
-        """
-        Check if the project fulfills the following criteria for
-        pushing the project result to FederatedCode.
-        """
-        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
+    def fetch_checkpoint_and_debian_index(self):
+        self.checkpoint_config_repo = federatedcode.clone_repository(
+            repo_url=self.pipeline_config_repo,
+            clone_path=self.working_path / "minecode-pipelines-config",
+            logger=self.log,
+        )
+        checkpoint = pipes.get_checkpoint_from_file(
+            cloned_repo=self.checkpoint_config_repo,
+            path=self.checkpoint_path,
+        )
 
-    def collect_packages_from_debian(self):
-        self.repos = debian.collect_packages_from_debian(logger=self.log)
+        self.last_checkpoint = checkpoint.get("previous_debian_index_last_modified_date")
+        self.log(f"last_checkpoint: {self.last_checkpoint}")
+        self.debian_collector = debian.DebianCollector(logger=self.log)
 
-    def delete_cloned_repos(self):
-        pipes.delete_cloned_repos(repos=self.repos, logger=self.log)
+    def mine_and_publish_alpine_packageurls(self):
+        _mine_and_publish_packageurls(
+            packageurls=self.debian_collector.get_packages(
+                previous_index_last_modified_date=self.last_checkpoint,
+            ),
+            total_package_count=None,
+            data_cluster=self.data_cluster,
+            checked_out_repos=self.checked_out_repos,
+            working_path=self.working_path,
+            append_purls=self.append_purls,
+            commit_msg_func=self.commit_message,
+            logger=self.log,
+        )
+
+    def save_check_point(self):
+        """Save Debian checkpoint only after successful completion of PURL mining"""
+        from commoncode.date import get_file_mtime
+
+        last_modified = get_file_mtime(self.debian_collector.index_location)
+        checkpoint = {"previous_debian_index_last_modified_date": last_modified}
+        self.log(f"Saving checkpoint: {checkpoint}")
+        pipes.update_checkpoints_in_github(
+            checkpoint=checkpoint,
+            cloned_repo=self.pipeline_config_repo,
+            path=self.checkpoint_path,
+            logger=self.log,
+        )

minecode_pipelines/pipes/debian.py

@@ -21,32 +21,21 @@
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 import gzip
-from datetime import datetime
 import logging
+from datetime import datetime
 from shutil import rmtree
+from traceback import format_exc as traceback_format_exc
 
 import debian_inspector
-from aboutcode import hashid
 from commoncode import fileutils
-from commoncode.date import get_file_mtime
 from packagedcode.models import PackageData
 from packageurl import PackageURL
-from scanpipe.pipes import federatedcode
 from scanpipe.pipes.fetch import fetch_http
 
-from minecode_pipelines import pipes
-from minecode_pipelines import VERSION
 from minecode_pipelines.pipes import ls
-from traceback import format_exc as traceback_format_exc
 
-DEBIAN_CHECKPOINT_PATH = "debian/checkpoints.json"
 DEBIAN_LSLR_URL = "http://ftp.debian.org/debian/ls-lR.gz"
 
-# We are testing and storing mined packageURLs in one single repo per ecosystem for now
-MINECODE_DATA_DEBIAN_REPO = "https://github.com/aboutcode-data/minecode-data-debian-test"
-
-PACKAGE_BATCH_SIZE = 1000
-
 
 def is_collectible(file_name):
     """Return True if a `file_name` is collectible."""
@@ -173,91 +162,4 @@ def get_packages(self, previous_index_last_modified_date=None, logger=None):
                 size=entry.size,
                 download_url=url_template.format(path=path),
             )
-            yield versionless_purl, packaged_data
-
-
-def commit_message(commit_batch, total_commit_batch="many"):
-    from django.conf import settings
-
-    author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME
-    author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL
-    tool_name = "pkg:github/aboutcode-org/scancode.io"
-
-    return f"""\
-        Collect PackageURLs from Debian ({commit_batch}/{total_commit_batch})
-
-        Tool: {tool_name}@v{VERSION}
-        Reference: https://{settings.ALLOWED_HOSTS[0]}
-
-        Signed-off-by: {author_name} <{author_email}>
-        """
-
-
-def collect_packages_from_debian(files_per_commit=PACKAGE_BATCH_SIZE, logger=None):
-    # Clone data and config repo
-    data_repo = federatedcode.clone_repository(
-        repo_url=MINECODE_DATA_DEBIAN_REPO,
-        logger=logger,
-    )
-    config_repo = federatedcode.clone_repository(
-        repo_url=pipes.MINECODE_PIPELINES_CONFIG_REPO,
-        logger=logger,
-    )
-    if logger:
-        logger(f"{MINECODE_DATA_DEBIAN_REPO} repo cloned at: {data_repo.working_dir}")
-        logger(f"{pipes.MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {config_repo.working_dir}")
-
-    # get last_modified to see if we can skip files
-    checkpoint = pipes.get_checkpoint_from_file(
-        cloned_repo=config_repo, path=DEBIAN_CHECKPOINT_PATH
-    )
-    last_modified = checkpoint.get("previous_debian_index_last_modified_date")
-    if logger:
-        logger(f"previous_debian_index_last_modified_date: {last_modified}")
-
-    # download and iterate through debian index
-    debian_collector = DebianCollector()
-    files_to_commit = []
-    commit_batch = 1
-    for current_purl, package in debian_collector.get_packages(
-        previous_index_last_modified_date=last_modified
-    ):
-        # write packageURL to file
-        package_base_dir = hashid.get_package_base_dir(purl=current_purl)
-        purl_file = pipes.write_packageurls_to_file(
-            repo=data_repo,
-            base_dir=package_base_dir,
-            packageurls=[package.purl],
-            append=True,
-        )
-        if purl_file not in files_to_commit:
-            files_to_commit.append(purl_file)
-
-        if len(files_to_commit) == files_per_commit:
-            federatedcode.commit_and_push_changes(
-                commit_message=commit_message(commit_batch),
-                repo=data_repo,
-                files_to_commit=files_to_commit,
-                logger=logger,
-            )
-            files_to_commit.clear()
-            commit_batch += 1
-
-    if files_to_commit:
-        federatedcode.commit_and_push_changes(
-            commit_message=commit_message(commit_batch),
-            repo=data_repo,
-            files_to_commit=files_to_commit,
-            logger=logger,
-        )
-
-    last_modified = get_file_mtime(debian_collector.index_location)
-    checkpoint = {"previous_debian_index_last_modified_date": last_modified}
-    if logger:
-        logger(f"checkpoint: {checkpoint}")
-    pipes.update_checkpoints_in_github(
-        checkpoint=checkpoint, cloned_repo=config_repo, path=DEBIAN_CHECKPOINT_PATH
-    )
-
-    repos_to_clean = [data_repo, config_repo]
-    return repos_to_clean
+            yield versionless_purl, [packaged_data.purl]

pyproject-minecode_pipelines.toml

@@ -4,7 +4,7 @@ build-backend = "flot.buildapi"
 
 [project]
 name = "minecode_pipelines"
-version = "0.0.1b42"
+version = "0.0.1b53"
 description = "A library for mining packageURLs and package metadata from ecosystem repositories."
 readme = "minecode_pipelines/README.rst"
 license = { text = "Apache-2.0" }