Adjust batch size

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

minecode_pipelines/pipelines/__init__.py

@@ -193,8 +193,6 @@ def _mine_and_publish_packageurls(
         )
         checkout["file_to_commit"].add(purl_file)
         checkout["file_processed_count"] += 1
-        if logger:
-            logger(f"{checkout['repo'].working_dir}: {checkout['file_processed_count']} / {batch_size}")
 
         if len(checkout["file_to_commit"]) > batch_size:
             if logger:

minecode_pipelines/pipelines/mine_npm.py

@@ -31,7 +31,7 @@ class MineNPM(MineCodeBasePipeline):
     a FederatedCode repo.
     """
 
-    package_batch_size = 70
+    package_batch_size = 5
 
     @classmethod
     def steps(cls):

minecode_pipelines/pipes/npm.py

@@ -228,6 +228,8 @@ def get_npm_packages_to_sync(packages_file, state, logger=None):
         if logger:
             logger(f"Starting package mining for {len(packages_to_sync)} packages")
 
+        synced_packages = []
+
     elif state == INITIAL_SYNC_STATE or state == PERIODIC_SYNC_STATE:
         synced_packages = get_mined_packages_from_checkpoint(
             config_repo=MINECODE_PIPELINES_CONFIG_REPO,