Merge remote-tracking branch 'origin/main' into weaver/marin-users-directory-for-output

# Conflicts:
#	lib/rigging/pyproject.toml
#	uv.lock

Author: rjpower

.agents/projects/2026-06-20_rigging_connection_auth.md

@@ -44,7 +44,8 @@ from any region.  The `mirrored()` wrapper marks an input path for
 marin prefix before the step runs.
 
 ```python
-from marin.execution.executor import mirrored, versioned
+from marin.execution.executor import mirrored
+from marin.execution.types import versioned
 
 step = ExecutorStep(
     name="train",

docs/explanations/executor.md

@@ -15,7 +15,7 @@ tpu_multislice = ResourceConfig.with_tpu("v4-8", slice_count=2)
 
 # GPU configuration
 gpu_config = ResourceConfig.with_gpu("H100", count=8)
-gpu_auto = ResourceConfig.with_gpu()  # auto-detect GPU type
+single_gpu = ResourceConfig.with_gpu("H100")  # count defaults to 1
 
 # CPU-only configuration
 cpu_config = ResourceConfig.with_cpu()

docs/references/resource-config.md

@@ -46,8 +46,8 @@ from marin.execution.executor import (
     ExecutorStep,
     executor_main,
     output_path_of,
-    this_output_path
 )
+from marin.execution.types import this_output_path
 ```
 
 Key imports:

docs/tutorials/executor-101.md

@@ -50,7 +50,7 @@ The tokenized dataset is a directory containing one file per shard of the datase
 
 ```python
 from levanter.data.text import TextLmDatasetFormat
-from marin.execution.executor import versioned
+from marin.execution.types import versioned
 from experiments.tokenization import default_tokenize
 from experiments.marin_models import marin_tokenizer
 

docs/tutorials/first-experiment.md

@@ -16,9 +16,7 @@ Start with the shared instructions in `/AGENTS.md`. Finelog-specific notes:
 - `src/finelog/store/` — `MemStore` (in-memory) and `DuckDBLogStore` (Parquet + DuckDB)
 - `src/finelog/server/` — `LogServiceImpl`, `StatsServiceImpl`, ASGI builder, CLI launcher
 - `src/finelog/client/` — `LogClient` (single user-facing entry; covers logs and stats),
-  `RemoteLogHandler`, error types in `errors.py`. `proxy.py` hosts
-  `LogServiceProxy`, an internal server-side adapter used when iris mounts the
-  log service as a forwarding proxy; not re-exported.
+  `RemoteLogHandler`, error types in `errors.py`.
 - `tests/` — store + server tests
 - `deploy/` — Dockerfile, k8s manifests, GCP snippets
 

lib/finelog/AGENTS.md

@@ -0,0 +1,29 @@
+# finelog log server for the CoreWeave US-EAST-02A cluster (cw-us-east-02a).
+#
+# Deployed as an in-cluster k8s Deployment + ClusterIP Service in the `iris`
+# namespace, the same namespace as the iris controller. iris references it from
+# its cluster config via `log_server_config: cw-us-east-02a`, which resolves to
+# the endpoint `k8s://finelog-cw-use02a.iris` ->
+# finelog-cw-use02a.iris.svc.cluster.local:10001.
+#
+# Archive: segments offload to Cloudflare R2 (s3://), the same object store the
+# cluster uses for controller state. `finelog deploy up` mints the
+# `finelog-cw-use02a-env` Secret from the operator's R2 creds + object_storage_endpoint
+# and projects it into the pod via envFrom (see deploy/k8s/02-deployment).
+#
+# Deploy: export KUBECONFIG=~/.kube/coreweave-iris-gpu and the R2 creds, then
+#   export R2_ACCESS_KEY_ID=... R2_SECRET_ACCESS_KEY=...
+#   uv run finelog deploy up cw-us-east-02a
+name: finelog-cw-use02a
+port: 10001
+image: ghcr.io/marin-community/finelog:latest
+remote_log_dir: s3://marin-na/finelog/cw-us-east-02a
+deployment:
+  k8s:
+    namespace: iris
+    # storage_class omitted -> the cluster's default StorageClass (shared-vast,
+    # VAST CSI) backs the PVC. Segments offload to R2, but size the local cache
+    # generously to absorb ingest bursts + L0->L1 compaction before offload.
+    storage_gb: 250
+    # R2 endpoint for the s3:// archive; folded into the minted creds Secret.
+    object_storage_endpoint: https://74981a43be0de7712369306c7b19133d.r2.cloudflarestorage.com

lib/finelog/config/cw-us-east-02a.yaml

@@ -2,6 +2,7 @@ name: finelog-marin
 port: 10001
 image: ghcr.io/marin-community/finelog:latest
 remote_log_dir: gs://marin-us-central2/finelog/marin
+client_url: iap+https://iris-marin.oa.dev/proxy/system.log-server
 deployment:
   gcp:
     project: hai-gcp-models

lib/finelog/config/marin.yaml

@@ -44,6 +44,15 @@ spec:
             - name: rpc
               containerPort: {{ port }}
               protocol: TCP
+          # S3 credentials for an s3:// remote_log_dir. `finelog deploy up` mints
+          # the `{{ name }}-env` Secret (R2 AWS_* creds + AWS_ENDPOINT_URL) from
+          # the operator's env and projects it here. optional=true keeps the pod
+          # schedulable when no Secret exists (gs:// or local archives, where GCS
+          # uses workload identity). The explicit env below wins on key collision.
+          envFrom:
+            - secretRef:
+                name: {{ name }}-env
+                optional: true
           env:
             - name: FINELOG_PORT
               value: "{{ port }}"