Adjust batch size

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

minecode_pipelines/__init__.py

@@ -8,4 +8,4 @@
 #
 
 
-VERSION = "0.0.1b61"
+VERSION = "0.0.1b62"

minecode_pipelines/pipelines/__init__.py

@@ -193,8 +193,6 @@ def _mine_and_publish_packageurls(
         )
         checkout["file_to_commit"].add(purl_file)
         checkout["file_processed_count"] += 1
-        if logger:
-            logger(f"{checkout['repo'].working_dir}: {checkout['file_processed_count']} / {batch_size}")
 
         if len(checkout["file_to_commit"]) > batch_size:
             if logger:

minecode_pipelines/pipelines/mine_npm.py

@@ -31,7 +31,7 @@ class MineNPM(MineCodeBasePipeline):
     a FederatedCode repo.
     """
 
-    package_batch_size = 70
+    package_batch_size = 5
 
     @classmethod
     def steps(cls):

minecode_pipelines/pipes/npm.py

@@ -228,6 +228,8 @@ def get_npm_packages_to_sync(packages_file, state, logger=None):
         if logger:
             logger(f"Starting package mining for {len(packages_to_sync)} packages")
 
+        synced_packages = []
+
     elif state == INITIAL_SYNC_STATE or state == PERIODIC_SYNC_STATE:
         synced_packages = get_mined_packages_from_checkpoint(
             config_repo=MINECODE_PIPELINES_CONFIG_REPO,

pyproject-minecode_pipelines.toml

@@ -4,7 +4,7 @@ build-backend = "flot.buildapi"
 
 [project]
 name = "minecode_pipelines"
-version = "0.0.1b61"
+version = "0.0.1b62"
 description = "A library for mining packageURLs and package metadata from ecosystem repositories."
 readme = "minecode_pipelines/README.rst"
 license = { text = "Apache-2.0" }
@@ -40,7 +40,7 @@ dependencies = [
     "aboutcode.hashid >= 0.2.0",
     "aboutcode.federated >= 0.1.0",
     "packageurl_python >= 0.15.6",
-    "scancodeio >= 36.0.1",
+    "scancodeio >= 35.3.0",
     "ftputil >= 5.1.0",
     "jawa >= 2.2.0",
     "arrow >= 1.3.0",
@@ -64,7 +64,7 @@ mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift"
 mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer"
 
 [tool.bumpversion]
-current_version = "0.0.1b61"
+current_version = "0.0.1b62"
 allow_dirty = true
 
 files = [