nemotron: set 4GB worker memory for CC download

Each Nemotron CC download worker decompresses a ~350MB zstd file to
~1.5-2GB in memory. The default ZephyrContext resources (1GB) caused
OOMKill when workers exceeded their memory limit. Set 4GB to give
sufficient headroom.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: yonromai

lib/marin/src/marin/download/nemotron_cc/download_nemotron_cc.py

@@ -20,6 +20,7 @@
 from marin.download.nemotron_cc.utils import decompress_zstd_stream
 from marin.execution import THIS_OUTPUT_PATH
 from marin.utils import fsspec_exists
+from fray.cluster import ResourceConfig
 from zephyr import Dataset, ZephyrContext
 from zephyr.writers import atomic_rename
 
@@ -102,7 +103,9 @@ def download_nemotron_cc(cfg: NemotronIngressConfig):
         .write_jsonl(os.path.join(cfg.output_path, ".metrics/download-{shard:05d}.jsonl"), skip_existing=True)
     )
 
-    ctx = ZephyrContext(name="download-nemotron-cc")
+    # Each worker downloads a ~350MB zstd file and decompresses to ~1.5-2GB in memory.
+    # Default ZephyrContext resources (1GB) causes OOMKill; 4GB gives sufficient headroom.
+    ctx = ZephyrContext(name="download-nemotron-cc", resources=ResourceConfig(cpu=1, ram="4g"))
     ctx.execute(pipeline)
 
     logger.info(f"Downloaded Nemotron CC files to {cfg.output_path}")