Support corpus_name

lpi-tn · lpi-tn · commit c5b24d1d7048 · 2025-04-24T16:25:04.000+02:00
diff --git a/welearn_datastack/nodes_workflow/DocumentClassifier/generate_to_classify_batch.py b/welearn_datastack/nodes_workflow/DocumentClassifier/generate_to_classify_batch.py
@@ -33,6 +33,7 @@ def main() -> None:
     parallelism_threshold: int = int(os.getenv("PARALLELISM_THRESHOLD", 100))
     parallelism_max: int = int(os.getenv("PARALLELISM_URL_MAX", 15))
     batch_urls_directory: str = os.getenv("BATCH_URLS_DIRECTORY", "batch_urls")
+    corpus_name: str = os.getenv("PICK_CORPUS_NAME", "*")
     qty_max_str: str | None = os.getenv("PICK_QTY_MAX", None)
 
     qty_max: int | None = None
@@ -61,6 +62,7 @@ def main() -> None:
             qty_max=qty_max,
             process_titles=[Step.DOCUMENT_VECTORIZED],
             weighed_scope=WeighedScope.DOCUMENT,
+            corpus_name=corpus_name,
         )
     )
     logger.info("'%s' Docsids were retrieved", len(ids_to_batch))

Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,7 @@ def main() -> None:`
`33`	`33`	`parallelism_threshold: int = int(os.getenv("PARALLELISM_THRESHOLD", 100))`
`34`	`34`	`parallelism_max: int = int(os.getenv("PARALLELISM_URL_MAX", 15))`
`35`	`35`	`batch_urls_directory: str = os.getenv("BATCH_URLS_DIRECTORY", "batch_urls")`
	`36`	`+ corpus_name: str = os.getenv("PICK_CORPUS_NAME", "*")`
`36`	`37`	`qty_max_str: str \| None = os.getenv("PICK_QTY_MAX", None)`
`37`	`38`
`38`	`39`	`qty_max: int \| None = None`
`@@ -61,6 +62,7 @@ def main() -> None:`
`61`	`62`	`qty_max=qty_max,`
`62`	`63`	`process_titles=[Step.DOCUMENT_VECTORIZED],`
`63`	`64`	`weighed_scope=WeighedScope.DOCUMENT,`
	`65`	`+ corpus_name=corpus_name,`
`64`	`66`	`)`
`65`	`67`	`)`
`66`	`68`	`logger.info("'%s' Docsids were retrieved", len(ids_to_batch))`