Use more shards.

rjpower · rjpower · commit 3560ae9d0873 · 2026-04-23T16:21:04.000-07:00
diff --git a/experiments/dedup/fineweb_10bt_exact.py b/experiments/dedup/fineweb_10bt_exact.py
@@ -36,7 +36,7 @@ def build_steps() -> list[StepSpec]:
         fn=lambda op: dedup_exact_paragraph(
             input_paths=os.path.join(download.output_path, "sample/10BT"),
             output_path=op,
-            max_parallelism=4,
+            max_parallelism=128,
         ),
     )
     return [download, dedup_step]

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ def build_steps() -> list[StepSpec]:`
`36`	`36`	`fn=lambda op: dedup_exact_paragraph(`
`37`	`37`	`input_paths=os.path.join(download.output_path, "sample/10BT"),`
`38`	`38`	`output_path=op,`
`39`		`- max_parallelism=4,`
	`39`	`+ max_parallelism=128,`
`40`	`40`	`),`
`41`	`41`	`)`
`42`	`42`	`return [download, dedup_step]`