zephyr: refactor chunking, improve shuffle scalability (#3839)

* closes #3643
 * closes #3798
 * closes #3994

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: ravwojdyla

lib/zephyr/README.md

@@ -217,7 +217,7 @@ When you call `ctx.execute(dataset)`:
 
 ```python
 # 1. Create plan from dataset operations
-plan = compute_plan(dataset, hints)
+plan = compute_plan(dataset)
 
 # 2. Get or create coordinator + workers
 coordinator = self._get_or_create_coordinator()

lib/zephyr/src/zephyr/__init__.py

@@ -8,7 +8,7 @@
 from zephyr.dataset import Dataset, ShardInfo
 from zephyr.execution import WorkerContext, ZephyrContext, zephyr_worker_ctx
 from zephyr.expr import Expr, col, lit
-from zephyr.plan import ExecutionHint, compute_plan
+from zephyr.plan import compute_plan
 from zephyr.readers import InputFileSpec, load_file, load_jsonl, load_parquet, load_vortex, load_zip_members
 from zephyr.writers import atomic_rename, write_jsonl_file, write_levanter_cache, write_parquet_file, write_vortex_file
 
@@ -17,7 +17,6 @@
 
 __all__ = [
     "Dataset",
-    "ExecutionHint",
     "Expr",
     "InputFileSpec",
     "ShardInfo",